creator-editing-studio 1.3.5 → 1.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/init.mjs +18 -1
- package/bin/update.mjs +146 -0
- package/history.json +1 -0
- package/package.json +2 -1
- package/template/CLAUDE.md +292 -236
- package/template/STYLE.md +179 -179
- package/template/package.json +8 -3
- package/template/preview/README.md +34 -0
- package/template/preview/lib.mjs +177 -0
- package/template/preview/player.html +255 -0
- package/template/preview/server.mjs +161 -0
- package/template/preview/use.mjs +22 -0
- package/template/requirements.txt +27 -23
- package/template/scripts/align-script.py +138 -0
- package/template/scripts/check_pair.py +90 -90
- package/template/scripts/doctor.mjs +167 -141
- package/template/scripts/find_cuts.py +45 -45
- package/template/scripts/lib/languages.py +68 -0
- package/template/scripts/lib/media.mjs +228 -228
- package/template/scripts/refmotion.mjs +349 -0
- package/template/scripts/setup_models.py +89 -0
- package/template/scripts/stills.mjs +1 -1
- package/template/scripts/takes.mjs +424 -424
- package/template/scripts/track_head.py +157 -0
- package/template/scripts/transcribe.py +172 -103
- package/template/src/components/CommandCard.tsx +12 -11
- package/template/src/components/Cursor.tsx +153 -153
- package/template/src/components/HeadPin.tsx +42 -0
- package/template/src/compositions/BeatTest.tsx +16 -15
- package/template/src/lib/track.ts +67 -0
- package/template/src/lib/tweaks.tsx +98 -0
- package/template/templates/BRIEF.md +97 -97
- package/template/templates/STYLE.md +179 -179
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
"""Track a person's head through a clip, so graphics can ride with it.
|
|
2
|
+
|
|
3
|
+
Usage (from the project root):
|
|
4
|
+
python scripts/track_head.py <video> <first_frame> <last_frame> <out.json> [--check <image.jpg>]
|
|
5
|
+
|
|
6
|
+
Frames are SOURCE frames of <video>. If <out.json> already exists, the new stretch is merged into it (same
|
|
7
|
+
frames replaced), so one file per video can hold every tracked stretch.
|
|
8
|
+
|
|
9
|
+
Writes, per source frame, where the head is: centre x/y, size (relative to the first frame) and tilt, in the
|
|
10
|
+
video's own pixels. A composition reads it and pins text or graphics to the head (`src/lib/track.ts`).
|
|
11
|
+
|
|
12
|
+
How: the head is found once — on the first frame, from the subject's silhouette (the same matting model as
|
|
13
|
+
make_cutout.py; no face model to download, works on any footage). Then dozens of small points on the face
|
|
14
|
+
are followed frame to frame (optical flow). A point is kept only if tracking it backwards lands where it
|
|
15
|
+
started, so points that slide off the face are dropped and fresh ones are found when too few remain. The
|
|
16
|
+
path is smoothed with a centred window — it removes jitter without making the graphic lag behind the head.
|
|
17
|
+
|
|
18
|
+
--check draws the tracked box on frames across the range into one image, to confirm by eye.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import argparse
|
|
22
|
+
import json
|
|
23
|
+
import sys
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
|
|
26
|
+
import numpy as np
|
|
27
|
+
|
|
28
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
29
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent / "lib"))
|
|
30
|
+
import cv2 # noqa: E402
|
|
31
|
+
from make_cutout import load_segmenter, probe, read_frames, segment # noqa: E402
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def head_box(alpha: np.ndarray):
|
|
35
|
+
"""The head from a silhouette: the top of the person, as wide as the silhouette is in that band."""
|
|
36
|
+
h, w = alpha.shape
|
|
37
|
+
rows = np.where((alpha > 0.5).sum(axis=1) > w * 0.02)[0]
|
|
38
|
+
if not len(rows):
|
|
39
|
+
raise SystemExit("No person found in the first frame.")
|
|
40
|
+
top = int(rows[0])
|
|
41
|
+
band = int(h * 0.12) # a head is about an eighth of a vertical frame's height
|
|
42
|
+
cols = np.where((alpha[top : top + band] > 0.5).any(axis=0))[0]
|
|
43
|
+
x0, x1 = int(cols.min()), int(cols.max())
|
|
44
|
+
# The face sits below the crown: start a little lower so the points land on features, not hair.
|
|
45
|
+
return x0, top + int(band * 0.25), x1 - x0, int(band * 0.85)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def seed(gray, box):
|
|
49
|
+
x, y, w, h = box
|
|
50
|
+
mask = np.zeros_like(gray)
|
|
51
|
+
mask[max(0, y) : y + h, max(0, x) : x + w] = 255
|
|
52
|
+
pts = cv2.goodFeaturesToTrack(gray, maxCorners=120, qualityLevel=0.01, minDistance=6, mask=mask)
|
|
53
|
+
return pts if pts is not None else np.empty((0, 1, 2), np.float32)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def smooth(values, win):
|
|
57
|
+
"""Centred moving average with edge padding — no lag, no shrinking at the ends."""
|
|
58
|
+
v = np.asarray(values, np.float64)
|
|
59
|
+
pad = win // 2
|
|
60
|
+
p = np.pad(v, pad, mode="edge")
|
|
61
|
+
return np.convolve(p, np.ones(win) / win, mode="valid")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def main() -> None:
|
|
65
|
+
# A Windows console is not UTF-8 by default; printing → or … would crash after the work is done.
|
|
66
|
+
for stream in (sys.stdout, sys.stderr):
|
|
67
|
+
try:
|
|
68
|
+
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
69
|
+
except (AttributeError, ValueError):
|
|
70
|
+
pass
|
|
71
|
+
ap = argparse.ArgumentParser()
|
|
72
|
+
ap.add_argument("video")
|
|
73
|
+
ap.add_argument("first", type=int)
|
|
74
|
+
ap.add_argument("last", type=int)
|
|
75
|
+
ap.add_argument("out")
|
|
76
|
+
ap.add_argument("--check", default=None)
|
|
77
|
+
ap.add_argument("--fresh", action="store_true", help="replace the file instead of merging into it")
|
|
78
|
+
args = ap.parse_args()
|
|
79
|
+
|
|
80
|
+
w, h, fps = probe(args.video)
|
|
81
|
+
count = args.last - args.first + 1
|
|
82
|
+
frames = list(read_frames(args.video, args.first, count, w, h, fps))
|
|
83
|
+
if not frames:
|
|
84
|
+
raise SystemExit("Could not read those frames.")
|
|
85
|
+
|
|
86
|
+
torch, model, device = load_segmenter("lite")
|
|
87
|
+
box = head_box(segment(torch, model, device, frames[0]))
|
|
88
|
+
bx, by, bw, bh = box
|
|
89
|
+
|
|
90
|
+
lk = dict(winSize=(21, 21), maxLevel=3, criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 30, 0.01))
|
|
91
|
+
prev = cv2.cvtColor(frames[0], cv2.COLOR_RGB2GRAY)
|
|
92
|
+
pts = seed(prev, box)
|
|
93
|
+
cx, cy, scale, angle = bx + bw / 2, by + bh / 2, 1.0, 0.0
|
|
94
|
+
track = [(cx, cy, scale, angle)]
|
|
95
|
+
for rgb in frames[1:]:
|
|
96
|
+
gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY)
|
|
97
|
+
if len(pts) >= 6:
|
|
98
|
+
nxt, st, _ = cv2.calcOpticalFlowPyrLK(prev, gray, pts, None, **lk)
|
|
99
|
+
back, st2, _ = cv2.calcOpticalFlowPyrLK(gray, prev, nxt, None, **lk)
|
|
100
|
+
ok = (st.ravel() == 1) & (st2.ravel() == 1) & (np.linalg.norm((back - pts).reshape(-1, 2), axis=1) < 1.0)
|
|
101
|
+
a, b = pts[ok], nxt[ok]
|
|
102
|
+
if len(a) >= 6:
|
|
103
|
+
# Similarity transform (move, scale, rotate) of the face from the last frame to this one.
|
|
104
|
+
m, _ = cv2.estimateAffinePartial2D(a, b, method=cv2.RANSAC, ransacReprojThreshold=2.0)
|
|
105
|
+
if m is not None:
|
|
106
|
+
s = float(np.hypot(m[0, 0], m[1, 0]))
|
|
107
|
+
r = float(np.degrees(np.arctan2(m[1, 0], m[0, 0])))
|
|
108
|
+
cx, cy = m[0, 0] * cx + m[0, 1] * cy + m[0, 2], m[1, 0] * cx + m[1, 1] * cy + m[1, 2]
|
|
109
|
+
scale *= s
|
|
110
|
+
angle += r
|
|
111
|
+
pts = b.reshape(-1, 1, 2)
|
|
112
|
+
if len(pts) < 30:
|
|
113
|
+
# Too few points left: find fresh ones in the head's current box.
|
|
114
|
+
hw, hh = bw * scale / 2, bh * scale / 2
|
|
115
|
+
pts = seed(gray, (int(cx - hw), int(cy - hh), int(hw * 2), int(hh * 2)))
|
|
116
|
+
track.append((cx, cy, scale, angle))
|
|
117
|
+
prev = gray
|
|
118
|
+
|
|
119
|
+
xs, ys, ss, rs = (smooth([t[i] for t in track], win) for i, win in ((0, 7), (1, 7), (2, 11), (3, 11)))
|
|
120
|
+
out = {
|
|
121
|
+
"source": str(args.video),
|
|
122
|
+
"fps": fps,
|
|
123
|
+
"size": [w, h],
|
|
124
|
+
"first": args.first,
|
|
125
|
+
"box": {"w": bw, "h": bh},
|
|
126
|
+
"frames": [
|
|
127
|
+
{"f": args.first + i, "x": round(float(xs[i]), 1), "y": round(float(ys[i]), 1), "s": round(float(ss[i]), 4), "r": round(float(rs[i]), 2)}
|
|
128
|
+
for i in range(len(track))
|
|
129
|
+
],
|
|
130
|
+
}
|
|
131
|
+
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
|
|
132
|
+
if Path(args.out).exists() and not args.fresh:
|
|
133
|
+
old = json.loads(Path(args.out).read_text())
|
|
134
|
+
if old.get("source") == out["source"]:
|
|
135
|
+
mine = {f["f"] for f in out["frames"]}
|
|
136
|
+
out["frames"] = sorted([f for f in old["frames"] if f["f"] not in mine] + out["frames"], key=lambda f: f["f"])
|
|
137
|
+
out["first"] = out["frames"][0]["f"]
|
|
138
|
+
Path(args.out).write_text(json.dumps(out, indent=1))
|
|
139
|
+
span = lambda k: f"{min(f[k] for f in out['frames']):.1f}…{max(f[k] for f in out['frames']):.1f}"
|
|
140
|
+
print(f"tracked {len(track)} frames · head x {span('x')} · y {span('y')} · size {span('s')} · tilt {span('r')}° → {args.out}")
|
|
141
|
+
|
|
142
|
+
if args.check:
|
|
143
|
+
picks = np.linspace(0, len(frames) - 1, 8).astype(int)
|
|
144
|
+
tiles = []
|
|
145
|
+
for i in picks:
|
|
146
|
+
img = cv2.cvtColor(frames[i], cv2.COLOR_RGB2BGR).copy()
|
|
147
|
+
f = out["frames"][i]
|
|
148
|
+
hw, hh = bw * f["s"] / 2, bh * f["s"] / 2
|
|
149
|
+
cv2.rectangle(img, (int(f["x"] - hw), int(f["y"] - hh)), (int(f["x"] + hw), int(f["y"] + hh)), (0, 255, 255), 4)
|
|
150
|
+
cv2.circle(img, (int(f["x"]), int(f["y"])), 10, (0, 0, 255), -1)
|
|
151
|
+
cv2.putText(img, str(f["f"]), (20, 60), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 255, 255), 4)
|
|
152
|
+
tiles.append(cv2.resize(img, (w // 4, h // 4)))
|
|
153
|
+
cv2.imwrite(args.check, np.hstack(tiles))
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
if __name__ == "__main__":
|
|
157
|
+
main()
|
|
@@ -1,103 +1,172 @@
|
|
|
1
|
-
"""Word-level transcript for a clip: WhisperX transcription + wav2vec2 forced alignment.
|
|
2
|
-
|
|
3
|
-
Usage (from the project root):
|
|
4
|
-
python scripts/transcribe.py <media> [--language en] [--model small]
|
|
5
|
-
|
|
6
|
-
Writes work/<clip-name>/words.json. Models are cached under the studio's .models folder.
|
|
7
|
-
Words the aligner cannot place (often digits like "186%") keep start/end = null and must be
|
|
8
|
-
fixed against the script before building the spine.
|
|
9
|
-
"""
|
|
10
|
-
|
|
11
|
-
import argparse
|
|
12
|
-
import json
|
|
13
|
-
import os
|
|
14
|
-
import time
|
|
15
|
-
from pathlib import Path
|
|
16
|
-
|
|
17
|
-
import sys as _sys # noqa: E402
|
|
18
|
-
_sys.path.insert(0, str(Path(__file__).resolve().parent / "lib"))
|
|
19
|
-
from media import FFMPEG, use_certifi # noqa: E402
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
os.environ.setdefault("
|
|
25
|
-
os.environ
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
#
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
print(f"
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
print("
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
print("
|
|
47
|
-
print(" .
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
if
|
|
103
|
-
|
|
1
|
+
"""Word-level transcript for a clip: WhisperX transcription + wav2vec2 forced alignment.
|
|
2
|
+
|
|
3
|
+
Usage (from the project root):
|
|
4
|
+
python scripts/transcribe.py <media> [--language en] [--model small]
|
|
5
|
+
|
|
6
|
+
Writes work/<clip-name>/words.json. Models are cached under the studio's .models folder.
|
|
7
|
+
Words the aligner cannot place (often digits like "186%") keep start/end = null and must be
|
|
8
|
+
fixed against the script before building the spine.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import time
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
import sys as _sys # noqa: E402
|
|
18
|
+
_sys.path.insert(0, str(Path(__file__).resolve().parent / "lib"))
|
|
19
|
+
from media import FFMPEG, use_certifi # noqa: E402
|
|
20
|
+
from languages import lang, load_aligner, studio_language # noqa: E402
|
|
21
|
+
|
|
22
|
+
# Models are large; cache them inside the studio so they survive and are easy to find.
|
|
23
|
+
_MODELS = Path(__file__).resolve().parent.parent / ".models"
|
|
24
|
+
os.environ.setdefault("HF_HOME", str(_MODELS / "huggingface"))
|
|
25
|
+
os.environ.setdefault("TORCH_HOME", str(_MODELS / "torch"))
|
|
26
|
+
os.environ["PATH"] = str(Path(FFMPEG).parent) + os.pathsep + os.environ["PATH"]
|
|
27
|
+
|
|
28
|
+
# torch.hub fetches the word-alignment model with urllib, which on a python.org macOS
|
|
29
|
+
# build has no root certificates — transcription then dies with CERTIFICATE_VERIFY_FAILED
|
|
30
|
+
# after the slow part has already succeeded. Must run before torch is imported.
|
|
31
|
+
use_certifi()
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
import whisperx # noqa: E402 (must import after the cache/PATH/cert setup above)
|
|
35
|
+
except ModuleNotFoundError as exc: # pragma: no cover
|
|
36
|
+
# Run with the wrong interpreter and the bare error reads as "never installed", which
|
|
37
|
+
# sends you off to fix the wrong thing. Say which python is running and what to do.
|
|
38
|
+
_venv = Path(__file__).resolve().parent.parent / ".venv"
|
|
39
|
+
_venv_py = _venv / ("Scripts/python.exe" if os.name == "nt" else "bin/python3")
|
|
40
|
+
print(f"Cannot import {exc.name}.", file=_sys.stderr)
|
|
41
|
+
print(f" running: {_sys.executable} (python {_sys.version.split()[0]})", file=_sys.stderr)
|
|
42
|
+
if _venv_py.exists():
|
|
43
|
+
print(f" the studio's environment is at {_venv_py}", file=_sys.stderr)
|
|
44
|
+
print(" run the transcriber with that python, or use: npm run prep", file=_sys.stderr)
|
|
45
|
+
else:
|
|
46
|
+
print(" the studio has no .venv yet. Build one with python 3.11 or 3.12:", file=_sys.stderr)
|
|
47
|
+
print(" py -3.12 -m venv .venv", file=_sys.stderr)
|
|
48
|
+
print(" .venv/Scripts/python.exe -m pip install -r requirements.txt", file=_sys.stderr)
|
|
49
|
+
print(" see what you have with: npm run doctor", file=_sys.stderr)
|
|
50
|
+
raise SystemExit(2)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# Indian languages: the small model cannot do them. On Malayalam it ignored the speech, invented one
|
|
54
|
+
# English sentence and repeated it, then drifted into Devanagari (2026-10-17). large-v3 is the
|
|
55
|
+
# smallest model that transcribes these reliably, so it is the default for them.
|
|
56
|
+
INDIC = {"ml", "ta", "te", "kn", "hi", "bn", "mr", "gu", "pa", "or", "as", "ur", "ne", "si"}
|
|
57
|
+
|
|
58
|
+
# The script each language must come out in. A transcript mostly in another script is a model
|
|
59
|
+
# failure (translated, or a neighbouring language), never a usable result, so it stops here.
|
|
60
|
+
SCRIPTS = {
|
|
61
|
+
"ml": (0x0D00, 0x0D7F), "ta": (0x0B80, 0x0BFF), "te": (0x0C00, 0x0C7F), "kn": (0x0C80, 0x0CFF),
|
|
62
|
+
"hi": (0x0900, 0x097F), "mr": (0x0900, 0x097F), "ne": (0x0900, 0x097F), "bn": (0x0980, 0x09FF),
|
|
63
|
+
"as": (0x0980, 0x09FF), "gu": (0x0A80, 0x0AFF), "pa": (0x0A00, 0x0A7F), "or": (0x0B00, 0x0B7F),
|
|
64
|
+
"si": (0x0D80, 0x0DFF), "ur": (0x0600, 0x06FF),
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def script_share(text: str, language: str) -> float | None:
|
|
69
|
+
"""Share of letters in the language's own script (Latin words like brand names are allowed)."""
|
|
70
|
+
span = SCRIPTS.get(language)
|
|
71
|
+
if not span:
|
|
72
|
+
return None
|
|
73
|
+
letters = [c for c in text if c.isalpha()]
|
|
74
|
+
if not letters:
|
|
75
|
+
return 0.0
|
|
76
|
+
return sum(span[0] <= ord(c) <= span[1] for c in letters) / len(letters)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def main() -> None:
|
|
80
|
+
# Windows consoles default to cp1252, which cannot print Malayalam: the script used to finish
|
|
81
|
+
# the transcription and then crash on its own summary line.
|
|
82
|
+
for stream in (_sys.stdout, _sys.stderr):
|
|
83
|
+
try:
|
|
84
|
+
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
85
|
+
except (AttributeError, ValueError):
|
|
86
|
+
pass
|
|
87
|
+
|
|
88
|
+
parser = argparse.ArgumentParser()
|
|
89
|
+
parser.add_argument("media")
|
|
90
|
+
parser.add_argument("--language", default=None, help="e.g. en, hi, ml. Default: the language chosen at setup (studio.config.json), else detected.")
|
|
91
|
+
parser.add_argument("--project", default=None, help="Video project name: writes to work/<project>/<clip>/")
|
|
92
|
+
parser.add_argument("--model", default=None, help="Whisper size. Default: what the language needs (scripts/lib/languages.py)")
|
|
93
|
+
args = parser.parse_args()
|
|
94
|
+
|
|
95
|
+
media = Path(args.media)
|
|
96
|
+
out_dir = (Path("work") / args.project / media.stem) if args.project else (Path("work") / media.stem)
|
|
97
|
+
out_dir.mkdir(parents=True, exist_ok=True)
|
|
98
|
+
started = time.time()
|
|
99
|
+
|
|
100
|
+
if args.language is None:
|
|
101
|
+
args.language = studio_language()
|
|
102
|
+
info = lang(args.language) if args.language else None
|
|
103
|
+
if info:
|
|
104
|
+
args.language = info["code"] # Hinglish is transcribed as Hindi
|
|
105
|
+
if args.model is None:
|
|
106
|
+
args.model = info["whisper"] if info else "small"
|
|
107
|
+
audio = whisperx.load_audio(str(media))
|
|
108
|
+
model = whisperx.load_model(args.model, device="cpu", compute_type="int8", language=args.language)
|
|
109
|
+
# task="transcribe" always: without it a weak model may answer in English instead.
|
|
110
|
+
result = model.transcribe(audio, batch_size=4, language=args.language, task="transcribe")
|
|
111
|
+
language = result["language"]
|
|
112
|
+
|
|
113
|
+
align_model, metadata = load_aligner(whisperx, language)
|
|
114
|
+
if align_model is not None:
|
|
115
|
+
aligned = whisperx.align(result["segments"], align_model, metadata, audio, device="cpu", return_char_alignments=False)
|
|
116
|
+
else:
|
|
117
|
+
# No word-timing model for this language: spread each segment's words across its time, by length.
|
|
118
|
+
# Looser than real alignment — good enough to cut on, worth checking where captions must land exactly.
|
|
119
|
+
aligned = {"segments": []}
|
|
120
|
+
for seg in result["segments"]:
|
|
121
|
+
ws = seg["text"].split()
|
|
122
|
+
total = sum(len(w) for w in ws) or 1
|
|
123
|
+
t, words_ = seg["start"], []
|
|
124
|
+
for w in ws:
|
|
125
|
+
d = (seg["end"] - seg["start"]) * len(w) / total
|
|
126
|
+
words_.append({"word": w, "start": t, "end": t + d})
|
|
127
|
+
t += d
|
|
128
|
+
aligned["segments"].append({**seg, "words": words_})
|
|
129
|
+
|
|
130
|
+
words = []
|
|
131
|
+
for segment in aligned["segments"]:
|
|
132
|
+
for w in segment.get("words", []):
|
|
133
|
+
words.append(
|
|
134
|
+
{
|
|
135
|
+
"word": w["word"],
|
|
136
|
+
"start": round(w["start"], 3) if "start" in w else None,
|
|
137
|
+
"end": round(w["end"], 3) if "end" in w else None,
|
|
138
|
+
"score": round(w["score"], 3) if "score" in w else None,
|
|
139
|
+
}
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
output = {
|
|
143
|
+
"source": str(media),
|
|
144
|
+
"language": language,
|
|
145
|
+
"model": args.model,
|
|
146
|
+
"durationSec": round(len(audio) / 16000, 3),
|
|
147
|
+
"text": " ".join(s["text"].strip() for s in aligned["segments"]),
|
|
148
|
+
"words": words,
|
|
149
|
+
}
|
|
150
|
+
out_file = out_dir / "words.json"
|
|
151
|
+
out_file.write_text(json.dumps(output, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
152
|
+
|
|
153
|
+
share = script_share(output["text"], language)
|
|
154
|
+
# Low bar on purpose: everyday Malayalam mixes in English words ("Instagram", "business"), but a
|
|
155
|
+
# failed run scores near zero (the small model's Malayalam came out at 0%).
|
|
156
|
+
if share is not None and share < 0.3:
|
|
157
|
+
print(
|
|
158
|
+
f"WARNING: only {share:.0%} of this transcript is in the {language} script, so the model did not "
|
|
159
|
+
f"really transcribe it (it translated, or heard another language). Do not cut from it. "
|
|
160
|
+
f"Re-run with a larger --model, and check --language is right.",
|
|
161
|
+
file=_sys.stderr,
|
|
162
|
+
)
|
|
163
|
+
raise SystemExit(3)
|
|
164
|
+
|
|
165
|
+
unplaced = [w["word"] for w in words if w["start"] is None]
|
|
166
|
+
print(f"language: {language} | words: {len(words)} | unplaced: {len(unplaced)} {unplaced if unplaced else ''}")
|
|
167
|
+
print(f"text: {output['text']}")
|
|
168
|
+
print(f"wrote {out_file} in {time.time() - started:.0f}s")
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
if __name__ == "__main__":
|
|
172
|
+
main()
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import React from 'react';
|
|
2
2
|
import {useCurrentFrame, useVideoConfig} from 'remotion';
|
|
3
|
-
import {
|
|
3
|
+
import {withAlpha} from '../design/color';
|
|
4
|
+
import {COLOR, FONT, RADIUS, SHADOW, SPACE, TYPE, WEIGHT} from '../design/tokens';
|
|
4
5
|
import {EASE} from '../design/motion';
|
|
5
6
|
import {progress} from '../lib/animate';
|
|
6
7
|
|
|
@@ -38,8 +39,8 @@ export type Segment =
|
|
|
38
39
|
| {kind: 'redacted'; text: string; label?: string};
|
|
39
40
|
|
|
40
41
|
const COLOR_FOR: Record<Segment['kind'], string> = {
|
|
41
|
-
plain:
|
|
42
|
-
flag:
|
|
42
|
+
plain: COLOR.fgOnDark,
|
|
43
|
+
flag: COLOR.fgMuted,
|
|
43
44
|
accent: COLOR.accent,
|
|
44
45
|
redacted: 'transparent',
|
|
45
46
|
};
|
|
@@ -59,8 +60,8 @@ const Redaction: React.FC<{width: number; label?: string}> = ({width, label}) =>
|
|
|
59
60
|
verticalAlign: 'middle',
|
|
60
61
|
borderRadius: RADIUS.sm / 2,
|
|
61
62
|
// Accent at low alpha, not a grey smear: it reads as a deliberate mark rather than a mistake.
|
|
62
|
-
backgroundColor:
|
|
63
|
-
border: `2px solid ${
|
|
63
|
+
backgroundColor: withAlpha(COLOR.accent, 0.18),
|
|
64
|
+
border: `2px solid ${withAlpha(COLOR.accent, 0.4)}`,
|
|
64
65
|
}}
|
|
65
66
|
>
|
|
66
67
|
<span
|
|
@@ -113,7 +114,7 @@ export const CommandCard: React.FC<{
|
|
|
113
114
|
width,
|
|
114
115
|
maxWidth: '100%',
|
|
115
116
|
borderRadius: RADIUS.md,
|
|
116
|
-
backgroundColor:
|
|
117
|
+
backgroundColor: COLOR.surfaceDark,
|
|
117
118
|
boxShadow: SHADOW.pop,
|
|
118
119
|
overflow: 'hidden',
|
|
119
120
|
opacity: land,
|
|
@@ -126,11 +127,11 @@ export const CommandCard: React.FC<{
|
|
|
126
127
|
alignItems: 'center',
|
|
127
128
|
gap: SPACE[2],
|
|
128
129
|
padding: `${SPACE[2]}px ${SPACE[3]}px`,
|
|
129
|
-
backgroundColor:
|
|
130
|
-
borderBottom: `2px solid ${
|
|
130
|
+
backgroundColor: COLOR.borderStrong,
|
|
131
|
+
borderBottom: `2px solid ${withAlpha(COLOR.fgOnDark, 0.08)}`,
|
|
131
132
|
}}
|
|
132
133
|
>
|
|
133
|
-
{[
|
|
134
|
+
{[COLOR.fgSecondary, COLOR.fgSecondary, COLOR.fgSecondary].map((c, i) => (
|
|
134
135
|
<span key={i} style={{width: 14, height: 14, borderRadius: RADIUS.pill, backgroundColor: c}} />
|
|
135
136
|
))}
|
|
136
137
|
<span style={{flex: 1}} />
|
|
@@ -141,7 +142,7 @@ export const CommandCard: React.FC<{
|
|
|
141
142
|
fontWeight: WEIGHT.medium,
|
|
142
143
|
letterSpacing: TYPE.eyebrow.tracking,
|
|
143
144
|
textTransform: 'uppercase',
|
|
144
|
-
color: note ? COLOR.accent :
|
|
145
|
+
color: note ? COLOR.accent : COLOR.fgMuted,
|
|
145
146
|
}}
|
|
146
147
|
>
|
|
147
148
|
{note ?? (platform === 'mac' ? 'macOS' : platform === 'windows' ? 'Windows' : '')}
|
|
@@ -155,7 +156,7 @@ export const CommandCard: React.FC<{
|
|
|
155
156
|
fontSize,
|
|
156
157
|
lineHeight: 1.65,
|
|
157
158
|
letterSpacing: '-0.01em',
|
|
158
|
-
color:
|
|
159
|
+
color: COLOR.fgOnDark,
|
|
159
160
|
wordBreak: 'break-word',
|
|
160
161
|
}}
|
|
161
162
|
>
|