creator-editing-studio 1.3.5 → 1.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,161 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * The live preview inside a Claude chat (an MCP App) — part of the studio.
4
+ *
5
+ * WHY THE PAGE CARRIES EVERYTHING. In the Claude desktop app the in-chat player cannot call this server
6
+ * back ("IPC not available for MCP tool call", tested 2026-10-17) and cannot fetch from the computer
7
+ * ("Failed to fetch", 2026-10-16). What it can do: show the page this server hands over when the preview
8
+ * opens, and send messages into the chat. So each time the preview opens, the page arrives with the
9
+ * compiled edit and small preview copies of the footage already inside it.
10
+ *
11
+ * THE LOOP. The person types a change under a beat → the player sends it into the chat → Claude edits
12
+ * that beat → Claude calls preview_edit again → the updated preview opens below. Every open saves a copy
13
+ * of the video's folder first, so preview_undo can put the last change back exactly.
14
+ *
15
+ * Which video: preview_edit({video}) — or current.json (written by use.mjs) when it is called without one. Nothing listens on any port. stdout belongs to the MCP
16
+ * protocol: every log goes to stderr.
17
+ */
18
+ import {cpSync, existsSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync} from 'node:fs';
19
+ import {join} from 'node:path';
20
+ import {McpServer} from '@modelcontextprotocol/sdk/server/mcp.js';
21
+ import {StdioServerTransport} from '@modelcontextprotocol/sdk/server/stdio.js';
22
+ import {registerAppResource, registerAppTool, RESOURCE_MIME_TYPE} from '@modelcontextprotocol/ext-apps/server';
23
+ import {z} from 'zod';
24
+ import {buildManifest, compileEdit, configFor, HERE, playerHtml, SRC} from './lib.mjs';
25
+
26
+ const CONFIG = join(HERE, 'current.json');
27
+ const cfg = () => JSON.parse(readFileSync(CONFIG, 'utf8'));
28
+ const log = (...a) => console.error('[live-preview]', ...a);
29
+ const SNAPS = (project) => join(HERE, '.snapshots', project);
30
+ const videoDir = (project) => join(SRC, 'videos', project);
31
+
32
+ /* ----------------------------------------------------------------------------- versions (for Undo) */
33
+
34
+ /** A fingerprint of the video's folder, so opening the preview twice without a change saves nothing. */
35
+ const fingerprint = (dir) =>
36
+ readdirSync(dir, {recursive: true})
37
+ .map(String)
38
+ .sort()
39
+ .map((f) => {
40
+ const st = statSync(join(dir, f));
41
+ return st.isFile() ? `${f}:${st.size}:${st.mtimeMs}` : '';
42
+ })
43
+ .join('|');
44
+ const sigFile = (project, v) => join(SNAPS(project), String(v), '.sig');
45
+
46
+ const versions = (project) =>
47
+ existsSync(SNAPS(project))
48
+ ? readdirSync(SNAPS(project))
49
+ .map(Number)
50
+ .filter((v) => Number.isFinite(v) && existsSync(sigFile(project, v)))
51
+ .sort((a, b) => a - b)
52
+ : [];
53
+
54
+ /** Save the video's folder as the next version if it changed since the last one. Returns the current version. */
55
+ function saveVersion(project) {
56
+ const dir = videoDir(project);
57
+ const list = versions(project);
58
+ const last = list.at(-1);
59
+ const sig = fingerprint(dir);
60
+ if (last !== undefined && readFileSync(sigFile(project, last), 'utf8') === sig) return last;
61
+ const v = (last ?? 0) + 1;
62
+ cpSync(dir, join(SNAPS(project), String(v)), {recursive: true});
63
+ // The fingerprint of the live folder as it was saved (the copy's own file dates differ).
64
+ writeFileSync(sigFile(project, v), sig);
65
+ for (const old of list.slice(0, Math.max(0, list.length - 40))) rmSync(join(SNAPS(project), String(old)), {recursive: true, force: true});
66
+ return v;
67
+ }
68
+
69
+ /* ----------------------------------------------------------------------------- the page */
70
+
71
+ let uiHtml = '';
72
+ let uiSig = '';
73
+ const media = new Map(); // hash → data URL, so unchanged footage is encoded once
74
+
75
+ /** The player page with the current edit and its footage inside it. */
76
+ async function page() {
77
+ const c = cfg();
78
+ const UI = join(HERE, 'ui');
79
+ const sig = existsSync(UI) ? readdirSync(UI).map((f) => `${f}:${statSync(join(UI, f)).mtimeMs}`).join('|') : 'built';
80
+ if (sig !== uiSig) [uiHtml, uiSig] = [await playerHtml(), sig];
81
+
82
+ const version = saveVersion(c.project);
83
+ let code = null;
84
+ let error = null;
85
+ try {
86
+ code = await compileEdit(c);
87
+ } catch (e) {
88
+ error = String(e.message ?? e);
89
+ }
90
+ const files = {};
91
+ for (const a of buildManifest(c)) {
92
+ if (!media.has(a.hash)) {
93
+ // Remotion's font loader reads the type from the text after the last dot, so fonts get it as a fragment.
94
+ const ext = /\.(ttf|otf|woff2?)$/i.exec(a.name)?.[1];
95
+ media.set(a.hash, `data:${a.mime};base64,${readFileSync(a.file).toString('base64')}${ext ? `#.${ext}` : ''}`);
96
+ }
97
+ files[a.name] = media.get(a.hash);
98
+ }
99
+ const data = {version, canUndo: versions(c.project).length > 1, project: c.project, edit: c.edit, composition: c.composition, error, code, files};
100
+ const html = uiHtml.replace('<head>', `<head><script>window.__LP_DATA=${JSON.stringify(data).replace(/</g, '\\u003c')}</script>`);
101
+ log(`page for ${c.project} version ${version}: ${(html.length / 1e6).toFixed(1)} MB${error ? ' (the edit does not compile)' : ''}`);
102
+ return html;
103
+ }
104
+
105
+ /* ----------------------------------------------------------------------------- MCP */
106
+
107
+ const URI = 'ui://editing-os/live-preview.html';
108
+ const server = new McpServer({name: 'editing-os-live-preview', version: '0.1.0'});
109
+
110
+ registerAppTool(
111
+ server,
112
+ 'preview_edit',
113
+ {
114
+ title: 'Live preview',
115
+ description:
116
+ 'Show the live preview of the current edit inside the chat: the video plays with the beats beside it, and a change box under every beat. Nothing is rendered. ' +
117
+ 'OPEN IT YOURSELF as a step of the work — right after the person approves the plan, and again after every change you make to the edit — never wait to be asked. ' +
118
+ 'Changes typed in the preview arrive as chat messages: change only that beat in the edit file, then call preview_edit again so the updated preview appears. ' +
119
+ 'Render the final video only when they press "Render final video", and give the estimated render time only then.',
120
+ inputSchema: {video: z.string().optional().describe('The video folder name (src/videos/<video>). Needed the first time for each video.')},
121
+ _meta: {ui: {resourceUri: URI}},
122
+ },
123
+ async ({video}) => {
124
+ if (video) writeFileSync(CONFIG, JSON.stringify(configFor(video), null, 2) + '\n');
125
+ if (!existsSync(CONFIG)) return {content: [{type: 'text', text: 'Which video? Call preview_edit with {video: "<folder name in src/videos>"}.'}], isError: true};
126
+ const c = cfg();
127
+ const v = saveVersion(c.project);
128
+ return {content: [{type: 'text', text: `Live preview of ${c.project} (version ${v}) is showing in the chat.`}], structuredContent: {project: c.project, version: v}};
129
+ },
130
+ );
131
+
132
+ server.registerTool(
133
+ 'preview_undo',
134
+ {
135
+ title: 'Undo the last preview change',
136
+ description: 'Put the edit back exactly as it was before the last change. Use it when the person presses Undo in the preview, then call preview_edit to show it.',
137
+ },
138
+ async () => {
139
+ const c = cfg();
140
+ const list = versions(c.project);
141
+ if (list.length < 2) return {content: [{type: 'text', text: 'Nothing to undo — this is the first version of the edit.'}], isError: true};
142
+ const [prev, last] = list.slice(-2);
143
+ const dir = videoDir(c.project);
144
+ const from = join(SNAPS(c.project), String(prev));
145
+ for (const f of readdirSync(dir)) rmSync(join(dir, f), {recursive: true, force: true});
146
+ cpSync(from, dir, {recursive: true});
147
+ rmSync(join(dir, '.sig'), {force: true});
148
+ rmSync(join(SNAPS(c.project), String(last)), {recursive: true, force: true});
149
+ // The restored folder is version `prev` again; remember its new fingerprint so it is not saved twice.
150
+ writeFileSync(sigFile(c.project, prev), fingerprint(dir));
151
+ log(`undo: ${c.project} back to version ${prev}`);
152
+ return {content: [{type: 'text', text: `Undone — ${c.project} is back to version ${prev}. Call preview_edit to show it.`}]};
153
+ },
154
+ );
155
+
156
+ registerAppResource(server, 'Live preview', URI, {description: 'Player and beat breakdown for the current edit'}, async () => ({
157
+ contents: [{uri: URI, mimeType: RESOURCE_MIME_TYPE, text: await page(), _meta: {ui: {prefersBorder: false}}}],
158
+ }));
159
+
160
+ await server.connect(new StdioServerTransport());
161
+ log('MCP server ready');
@@ -0,0 +1,22 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * Point the live preview at a video (preview_edit({video}) does the same).
4
+ *
5
+ * node preview/use.mjs <name>
6
+ *
7
+ * Writes current.json from the video's own folder (see configFor in lib.mjs); the running preview server
8
+ * watches that file and switches over by itself. The online preview uses publish.mjs instead.
9
+ */
10
+ import {writeFileSync} from 'node:fs';
11
+ import {join} from 'node:path';
12
+ import {configFor, HERE} from './lib.mjs';
13
+
14
+ try {
15
+ const config = configFor(process.argv[2]);
16
+ if (!config.stems.voice) console.warn('warning: no voice stem found — the preview will play silent');
17
+ writeFileSync(join(HERE, 'current.json'), JSON.stringify(config, null, 2) + '\n');
18
+ console.log(JSON.stringify(config, null, 2));
19
+ } catch (e) {
20
+ console.error(String(e.message ?? e));
21
+ process.exit(1);
22
+ }
@@ -1,23 +1,27 @@
1
- # The Python half of the studio: transcription, word alignment, cut detection,
2
- # green-screen cutouts and colour grading.
3
- #
4
- # python -m venv .venv
5
- # .venv/bin/pip install -r requirements.txt (Windows: .venv\Scripts\pip)
6
- #
7
- # Python 3.11 or 3.12. Not 3.13+ — WhisperX's dependencies do not support it yet, and the
8
- # failure is a wall of build errors rather than a clear message.
9
- #
10
- # certifi is not optional on macOS: torch downloads the alignment model with urllib, which
11
- # on a python.org build has no root certificates at all.
12
-
13
- whisperx==3.8.6
14
- # BiRefNet, for cutting the subject out of ordinary footage so text can sit behind them.
15
- # MIT licensed and self-hosted: nothing uploads, and there is no per-video cost.
16
- transformers>=4.44
17
- timm>=1.0
18
- einops>=0.8
19
- kornia>=0.7
20
- torch>=2.2
21
- numpy>=1.26
22
- Pillow>=10.0
23
- certifi>=2024.2.2
1
+ # The Python half of the studio: transcription, word alignment, cut detection,
2
+ # green-screen cutouts and colour grading.
3
+ #
4
+ # python -m venv .venv
5
+ # .venv/bin/pip install -r requirements.txt (Windows: .venv\Scripts\pip)
6
+ #
7
+ # Python 3.11 or 3.12. Not 3.13+ — WhisperX's dependencies do not support it yet, and the
8
+ # failure is a wall of build errors rather than a clear message.
9
+ #
10
+ # certifi is not optional on macOS: torch downloads the alignment model with urllib, which
11
+ # on a python.org build has no root certificates at all.
12
+
13
+ whisperx==3.8.6
14
+ # BiRefNet, for cutting the subject out of ordinary footage so text can sit behind them.
15
+ # MIT licensed and self-hosted: nothing uploads, and there is no per-video cost.
16
+ transformers>=4.44
17
+ timm>=1.0
18
+ einops>=0.8
19
+ kornia>=0.7
20
+ torch>=2.2
21
+ numpy>=1.26
22
+ Pillow>=10.0
23
+ certifi>=2024.2.2
24
+
25
+ # Motion tracking (scripts/track_head.py): optical-flow tracking of the speaker's head, so text can ride
26
+ # with it. Headless build: no GUI libraries, works on servers and Macs alike. Only used when asked for.
27
+ opencv-python-headless>=4.9
@@ -0,0 +1,138 @@
1
+ """Word timings from a script you already have: match each word of an .srt to the audio.
2
+
3
+ Usage (from the project root):
4
+ python scripts/align-script.py <media> <script.srt> --language ml [--project name]
5
+
6
+ Writes work/<project>/<clip>/words.json in the same shape as transcribe.py, so everything downstream
7
+ works unchanged.
8
+
9
+ WHY THIS EXISTS. For Indian languages, speech recognition is the weak link: on a Malayalam take the
10
+ small Whisper model ignored the speech and invented English, and even large models misspell names and
11
+ loanwords. When the creator already has the script (or a corrected .srt), the words are known — only
12
+ their timing is not. Forced alignment (wav2vec2, per language) answers exactly that, and it is far more
13
+ reliable than recognition: it cannot invent words, it only places the ones it is given.
14
+
15
+ The .srt timings only need to be roughly right — each line is aligned inside its own window, padded a
16
+ little either side, so a line timed a few hundred milliseconds off still lands correctly.
17
+ """
18
+
19
+ import argparse
20
+ import json
21
+ import os
22
+ import re
23
+ import sys as _sys
24
+ import time
25
+ from pathlib import Path
26
+
27
+ _sys.path.insert(0, str(Path(__file__).resolve().parent / "lib"))
28
+ from media import FFMPEG, use_certifi # noqa: E402
29
+ from languages import lang, load_aligner, studio_language # noqa: E402
30
+
31
+ _MODELS = Path(__file__).resolve().parent.parent / ".models"
32
+ os.environ.setdefault("HF_HOME", str(_MODELS / "huggingface"))
33
+ os.environ.setdefault("TORCH_HOME", str(_MODELS / "torch"))
34
+ os.environ["PATH"] = str(Path(FFMPEG).parent) + os.pathsep + os.environ["PATH"]
35
+ use_certifi()
36
+
37
+ import whisperx # noqa: E402
38
+
39
+ TIME = re.compile(r"(\d+):(\d+):(\d+)[,.](\d+)\s*-->\s*(\d+):(\d+):(\d+)[,.](\d+)")
40
+
41
+
42
+ def read_srt(path: Path):
43
+ """Cues as {start, end, text}. Tolerates a missing end on the last cue (a cut-off file)."""
44
+ cues = []
45
+ for block in re.split(r"\n\s*\n", path.read_text(encoding="utf-8-sig").strip()):
46
+ lines = [l.strip() for l in block.splitlines() if l.strip()]
47
+ for i, line in enumerate(lines):
48
+ m = TIME.search(line)
49
+ if m:
50
+ h1, m1, s1, ms1, h2, m2, s2, ms2 = map(int, m.groups())
51
+ text = " ".join(lines[i + 1 :]).strip()
52
+ if text:
53
+ cues.append({"start": h1 * 3600 + m1 * 60 + s1 + ms1 / 1000, "end": h2 * 3600 + m2 * 60 + s2 + ms2 / 1000, "text": text})
54
+ break
55
+ return cues
56
+
57
+
58
+ def main() -> None:
59
+ for stream in (_sys.stdout, _sys.stderr):
60
+ try:
61
+ stream.reconfigure(encoding="utf-8", errors="replace")
62
+ except (AttributeError, ValueError):
63
+ pass
64
+
65
+ parser = argparse.ArgumentParser()
66
+ parser.add_argument("media")
67
+ parser.add_argument("srt")
68
+ parser.add_argument("--language", default=None, help="e.g. ml, hi, ta, en. Default: the language chosen at setup")
69
+ parser.add_argument("--project", default=None)
70
+ parser.add_argument("--whole", action="store_true", help="ignore the cue times; align the whole script against the whole clip")
71
+ parser.add_argument("--pad", type=float, default=0.6, help="seconds of slack around each cue")
72
+ args = parser.parse_args()
73
+
74
+ media = Path(args.media)
75
+ out_dir = (Path("work") / args.project / media.stem) if args.project else (Path("work") / media.stem)
76
+ out_dir.mkdir(parents=True, exist_ok=True)
77
+ started = time.time()
78
+
79
+ cues = read_srt(Path(args.srt))
80
+ if not cues:
81
+ raise SystemExit("no cues found in the .srt")
82
+ audio = whisperx.load_audio(str(media))
83
+ duration = len(audio) / 16000
84
+
85
+ # Pad each cue a little past its own times, even into a neighbour's window: hand-made .srt cues often
86
+ # start a few hundred ms after the speech does, and a window clipped to the cue pins those first
87
+ # words onto its edge (they show up with a score near 0 at exactly the cue start).
88
+ segments = []
89
+ if args.whole:
90
+ # One window over the whole clip: the cue times are not trusted at all, only the words and their order.
91
+ segments = [{"start": 0.0, "end": duration, "text": " ".join(c["text"] for c in cues)}]
92
+ for c in ([] if args.whole else cues):
93
+ lo = max(0.0, c["start"] - args.pad)
94
+ hi = min(duration, c["end"] + args.pad)
95
+ segments.append({"start": lo, "end": max(hi, lo + 0.2), "text": c["text"]})
96
+
97
+ args.language = lang(args.language or studio_language() or "en")["code"]
98
+ align_model, metadata = load_aligner(whisperx, args.language)
99
+ if align_model is None:
100
+ raise SystemExit(f"No word-timing model for '{args.language}', so a script cannot be matched to the audio. Transcribe instead: python scripts/transcribe.py <media>")
101
+ aligned = whisperx.align(segments, align_model, metadata, audio, device="cpu", return_char_alignments=False)
102
+
103
+ words = []
104
+ for segment in aligned["segments"]:
105
+ for w in segment.get("words", []):
106
+ words.append(
107
+ {
108
+ "word": w["word"],
109
+ "start": round(w["start"], 3) if "start" in w else None,
110
+ "end": round(w["end"], 3) if "end" in w else None,
111
+ "score": round(w["score"], 3) if "score" in w else None,
112
+ }
113
+ )
114
+
115
+ output = {
116
+ "source": str(media),
117
+ "language": args.language,
118
+ "model": f"forced-alignment:{metadata.get('language', args.language)} from {Path(args.srt).name}",
119
+ "durationSec": round(duration, 3),
120
+ "scriptCoversSec": round(cues[-1]["end"], 3),
121
+ "text": " ".join(c["text"] for c in cues),
122
+ "words": words,
123
+ }
124
+ out_file = out_dir / "words.json"
125
+ out_file.write_text(json.dumps(output, indent=2, ensure_ascii=False), encoding="utf-8")
126
+
127
+ unplaced = [w["word"] for w in words if w["start"] is None]
128
+ weak = [f'{w["word"]}@{w["start"]}' for w in words if w["score"] is not None and w["score"] < 0.15]
129
+ if weak:
130
+ print(f"low-confidence words (check by ear): {', '.join(weak)}")
131
+ print(f"cues: {len(cues)} | words: {len(words)} | unplaced: {len(unplaced)} {unplaced if unplaced else ''}")
132
+ if cues[-1]["end"] < duration - 1:
133
+ print(f"note: the script ends at {cues[-1]['end']:.1f}s but the video runs to {duration:.1f}s")
134
+ print(f"wrote {out_file} in {time.time() - started:.0f}s")
135
+
136
+
137
+ if __name__ == "__main__":
138
+ main()
@@ -1,90 +1,90 @@
1
- """Check that a green-screen clip is frame-aligned with its original.
2
-
3
- Usage (from the project root):
4
- python scripts/check_pair.py <original> <greenscreen> [--samples 7]
5
-
6
- At several points through the clip, compares the subject pixels of green-screen frame K with
7
- original frames K-2..K+2. The offset with the smallest difference is the alignment; it must be the
8
- same at every sample (otherwise the clips drift).
9
- """
10
-
11
- import argparse
12
- import json
13
- import subprocess
14
-
15
- import numpy as np
16
- import sys as _sys, pathlib as _pl
17
- _sys.path.insert(0, str(_pl.Path(__file__).resolve().parent / 'lib'))
18
- from media import FFMPEG, FFPROBE # resolved per platform - never hardcode a tool path
19
-
20
-
21
-
22
-
23
- OFFSETS = range(-2, 3)
24
-
25
-
26
- def probe(path):
27
- out = subprocess.run(
28
- [FFPROBE, "-v", "error", "-select_streams", "v:0", "-count_frames",
29
- "-show_entries", "stream=width,height,r_frame_rate,nb_read_frames", "-of", "json", path],
30
- capture_output=True, text=True, check=True,
31
- )
32
- s = json.loads(out.stdout)["streams"][0]
33
- num, den = s["r_frame_rate"].split("/")
34
- return int(s["width"]), int(s["height"]), float(num) / float(den), int(s["nb_read_frames"])
35
-
36
-
37
- def frames(path, start, count, width, height, fps):
38
- out = subprocess.run(
39
- [FFMPEG, "-v", "error", "-ss", f"{start / fps:.6f}", "-i", path, "-frames:v", str(count),
40
- "-f", "rawvideo", "-pix_fmt", "rgb24", "-"],
41
- capture_output=True, check=True,
42
- )
43
- data = np.frombuffer(out.stdout, dtype=np.uint8)
44
- return data.reshape(-1, height, width, 3).astype(np.float32)
45
-
46
-
47
- def main():
48
- parser = argparse.ArgumentParser()
49
- parser.add_argument("original")
50
- parser.add_argument("greenscreen")
51
- parser.add_argument("--samples", type=int, default=7)
52
- args = parser.parse_args()
53
-
54
- w1, h1, fps1, n1 = probe(args.original)
55
- w2, h2, fps2, n2 = probe(args.greenscreen)
56
- print(f"original: {w1}x{h1} @ {fps1:.3f}fps, {n1} frames")
57
- print(f"greenscreen: {w2}x{h2} @ {fps2:.3f}fps, {n2} frames")
58
- if (w1, h1) != (w2, h2) or abs(fps1 - fps2) > 1e-3:
59
- print("MISMATCH: resolution or frame rate differ")
60
- return
61
-
62
- last = min(n1, n2) - 3
63
- points = np.linspace(3, last, args.samples).astype(int)
64
- results = []
65
- for k in points:
66
- green = frames(args.greenscreen, k, 1, w2, h2, fps2)[0]
67
- r, g, b = green[..., 0], green[..., 1], green[..., 2]
68
- background = (g > 120) & (g > r * 1.35) & (g > b * 1.35)
69
- subject = ~background
70
- # Erode the subject mask so blended edge pixels don't bias the comparison.
71
- for _ in range(4):
72
- subject = subject & np.roll(subject, 1, 0) & np.roll(subject, -1, 0) & np.roll(subject, 1, 1) & np.roll(subject, -1, 1)
73
- coverage = subject.mean()
74
- originals = frames(args.original, k + OFFSETS.start, len(OFFSETS), w1, h1, fps1)
75
- diffs = {off: float(np.abs(originals[i][subject] - green[subject]).mean()) for i, off in enumerate(OFFSETS) if i < len(originals)}
76
- best = min(diffs, key=diffs.get)
77
- results.append(best)
78
- pretty = " ".join(f"{off:+d}:{d:5.2f}" for off, d in diffs.items())
79
- print(f"frame {k:5d} ({k / fps1:6.2f}s) subject {coverage:5.1%} best offset {best:+d} [{pretty}]")
80
-
81
- if len(set(results)) == 1:
82
- off = results[0]
83
- print(f"\nRESULT: aligned with a constant offset of {off:+d} frame(s)"
84
- + (" — perfect." if off == 0 else " — shift the green-screen clip to match."))
85
- else:
86
- print(f"\nRESULT: offsets vary {sorted(set(results))} — clips drift; re-export both from the same timeline.")
87
-
88
-
89
- if __name__ == "__main__":
90
- main()
1
+ """Check that a green-screen clip is frame-aligned with its original.
2
+
3
+ Usage (from the project root):
4
+ python scripts/check_pair.py <original> <greenscreen> [--samples 7]
5
+
6
+ At several points through the clip, compares the subject pixels of green-screen frame K with
7
+ original frames K-2..K+2. The offset with the smallest difference is the alignment; it must be the
8
+ same at every sample (otherwise the clips drift).
9
+ """
10
+
11
+ import argparse
12
+ import json
13
+ import subprocess
14
+
15
+ import numpy as np
16
+ import sys as _sys, pathlib as _pl
17
+ _sys.path.insert(0, str(_pl.Path(__file__).resolve().parent / 'lib'))
18
+ from media import FFMPEG, FFPROBE # resolved per platform - never hardcode a tool path
19
+
20
+
21
+
22
+
23
+ OFFSETS = range(-2, 3)
24
+
25
+
26
+ def probe(path):
27
+ out = subprocess.run(
28
+ [FFPROBE, "-v", "error", "-select_streams", "v:0", "-count_frames",
29
+ "-show_entries", "stream=width,height,r_frame_rate,nb_read_frames", "-of", "json", path],
30
+ capture_output=True, text=True, check=True,
31
+ )
32
+ s = json.loads(out.stdout)["streams"][0]
33
+ num, den = s["r_frame_rate"].split("/")
34
+ return int(s["width"]), int(s["height"]), float(num) / float(den), int(s["nb_read_frames"])
35
+
36
+
37
+ def frames(path, start, count, width, height, fps):
38
+ out = subprocess.run(
39
+ [FFMPEG, "-v", "error", "-ss", f"{start / fps:.6f}", "-i", path, "-frames:v", str(count),
40
+ "-f", "rawvideo", "-pix_fmt", "rgb24", "-"],
41
+ capture_output=True, check=True,
42
+ )
43
+ data = np.frombuffer(out.stdout, dtype=np.uint8)
44
+ return data.reshape(-1, height, width, 3).astype(np.float32)
45
+
46
+
47
+ def main():
48
+ parser = argparse.ArgumentParser()
49
+ parser.add_argument("original")
50
+ parser.add_argument("greenscreen")
51
+ parser.add_argument("--samples", type=int, default=7)
52
+ args = parser.parse_args()
53
+
54
+ w1, h1, fps1, n1 = probe(args.original)
55
+ w2, h2, fps2, n2 = probe(args.greenscreen)
56
+ print(f"original: {w1}x{h1} @ {fps1:.3f}fps, {n1} frames")
57
+ print(f"greenscreen: {w2}x{h2} @ {fps2:.3f}fps, {n2} frames")
58
+ if (w1, h1) != (w2, h2) or abs(fps1 - fps2) > 1e-3:
59
+ print("MISMATCH: resolution or frame rate differ")
60
+ return
61
+
62
+ last = min(n1, n2) - 3
63
+ points = np.linspace(3, last, args.samples).astype(int)
64
+ results = []
65
+ for k in points:
66
+ green = frames(args.greenscreen, k, 1, w2, h2, fps2)[0]
67
+ r, g, b = green[..., 0], green[..., 1], green[..., 2]
68
+ background = (g > 120) & (g > r * 1.35) & (g > b * 1.35)
69
+ subject = ~background
70
+ # Erode the subject mask so blended edge pixels don't bias the comparison.
71
+ for _ in range(4):
72
+ subject = subject & np.roll(subject, 1, 0) & np.roll(subject, -1, 0) & np.roll(subject, 1, 1) & np.roll(subject, -1, 1)
73
+ coverage = subject.mean()
74
+ originals = frames(args.original, k + OFFSETS.start, len(OFFSETS), w1, h1, fps1)
75
+ diffs = {off: float(np.abs(originals[i][subject] - green[subject]).mean()) for i, off in enumerate(OFFSETS) if i < len(originals)}
76
+ best = min(diffs, key=diffs.get)
77
+ results.append(best)
78
+ pretty = " ".join(f"{off:+d}:{d:5.2f}" for off, d in diffs.items())
79
+ print(f"frame {k:5d} ({k / fps1:6.2f}s) subject {coverage:5.1%} best offset {best:+d} [{pretty}]")
80
+
81
+ if len(set(results)) == 1:
82
+ off = results[0]
83
+ print(f"\nRESULT: aligned with a constant offset of {off:+d} frame(s)"
84
+ + (" — perfect." if off == 0 else " — shift the green-screen clip to match."))
85
+ else:
86
+ print(f"\nRESULT: offsets vary {sorted(set(results))} — clips drift; re-export both from the same timeline.")
87
+
88
+
89
+ if __name__ == "__main__":
90
+ main()