creator-editing-studio 1.3.5 → 1.3.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/init.mjs +18 -1
- package/bin/update.mjs +146 -0
- package/history.json +1 -0
- package/package.json +2 -1
- package/template/CLAUDE.md +292 -236
- package/template/STYLE.md +179 -179
- package/template/package.json +8 -3
- package/template/preview/README.md +34 -0
- package/template/preview/lib.mjs +177 -0
- package/template/preview/player.html +255 -0
- package/template/preview/server.mjs +161 -0
- package/template/preview/use.mjs +22 -0
- package/template/requirements.txt +27 -23
- package/template/scripts/align-script.py +138 -0
- package/template/scripts/check_pair.py +90 -90
- package/template/scripts/doctor.mjs +167 -141
- package/template/scripts/find_cuts.py +45 -45
- package/template/scripts/lib/languages.py +68 -0
- package/template/scripts/lib/media.mjs +228 -228
- package/template/scripts/refmotion.mjs +349 -0
- package/template/scripts/setup_models.py +89 -0
- package/template/scripts/stills.mjs +1 -1
- package/template/scripts/takes.mjs +424 -424
- package/template/scripts/track_head.py +157 -0
- package/template/scripts/transcribe.py +172 -103
- package/template/src/components/CommandCard.tsx +12 -11
- package/template/src/components/Cursor.tsx +153 -153
- package/template/src/components/HeadPin.tsx +42 -0
- package/template/src/compositions/BeatTest.tsx +16 -15
- package/template/src/lib/track.ts +67 -0
- package/template/src/lib/tweaks.tsx +98 -0
- package/template/templates/BRIEF.md +97 -97
- package/template/templates/STYLE.md +179 -179
|
@@ -0,0 +1,349 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* Study HOW a reference moves and sounds, not just what it looks like at a few moments.
|
|
4
|
+
*
|
|
5
|
+
* node scripts/refmotion.mjs <clip> [--from <s>] [--to <s>] [--strips 10] [--out work/refs/motion/<name>]
|
|
6
|
+
*
|
|
7
|
+
* A contact sheet shows where things end up. It cannot show the thing that makes motion feel expensive:
|
|
8
|
+
* how a move speeds up and slows down, how long it holds, whether one scene flows into the next or cuts,
|
|
9
|
+
* and whether the sound lands ahead of the picture or on it. This measures all of that, frame by frame:
|
|
10
|
+
*
|
|
11
|
+
* MOTION every frame's change from the one before. A hard cut is a one-frame spike; a move is a
|
|
12
|
+
* hill. Each hill is reported with its length, how many frames it takes to reach full speed
|
|
13
|
+
* and to settle (an eased move settles slowly — long tail), and the holds between moves.
|
|
14
|
+
* SOUND onsets (where a sound starts), each classed by its shape — click, hit, whoosh, other —
|
|
15
|
+
* plus loudness and, if there is a steady pulse, the music's tempo.
|
|
16
|
+
* SYNC for every cut and every move's peak, the nearest sound and how many frames it leads or lags.
|
|
17
|
+
* STRIPS every frame across each cut and the biggest moves, as images to look at closely.
|
|
18
|
+
*
|
|
19
|
+
* Writes report.md + report.json + strips/*.jpg. Measurement only — nothing from a reference is ever
|
|
20
|
+
* used in a video.
|
|
21
|
+
*/
|
|
22
|
+
import {spawnSync} from 'node:child_process';
|
|
23
|
+
import {mkdirSync, rmSync, writeFileSync} from 'node:fs';
|
|
24
|
+
import {basename, extname, join} from 'node:path';
|
|
25
|
+
import {FFMPEG, probe} from './lib/media.mjs';
|
|
26
|
+
|
|
27
|
+
const args = process.argv.slice(2);
|
|
28
|
+
const clip = args.find((a) => !a.startsWith('--') && !/^\d/.test(a));
|
|
29
|
+
const flag = (n, d) => (args.includes(`--${n}`) ? args[args.indexOf(`--${n}`) + 1] : d);
|
|
30
|
+
if (!clip) {
|
|
31
|
+
console.error('Usage: node scripts/refmotion.mjs <clip> [--from s] [--to s] [--strips 10]');
|
|
32
|
+
process.exit(2);
|
|
33
|
+
}
|
|
34
|
+
const name = basename(clip, extname(clip));
|
|
35
|
+
const out = flag('out', join('work', 'refs', 'motion', name));
|
|
36
|
+
const from = Number(flag('from', 0));
|
|
37
|
+
const vs = probe(clip);
|
|
38
|
+
if (!vs) throw new Error(`cannot read ${clip}`);
|
|
39
|
+
const fps = vs.fps;
|
|
40
|
+
const total = vs.duration;
|
|
41
|
+
const to = Math.min(total, Number(flag('to', Math.min(total, from + 180))));
|
|
42
|
+
const maxStrips = Number(flag('strips', 10));
|
|
43
|
+
rmSync(out, {recursive: true, force: true});
|
|
44
|
+
mkdirSync(join(out, 'strips'), {recursive: true});
|
|
45
|
+
|
|
46
|
+
const range = ['-ss', String(from), '-to', String(to)];
|
|
47
|
+
const tc = (s) => `${Math.floor(s / 60)}:${(s % 60).toFixed(2).padStart(5, '0')}`;
|
|
48
|
+
|
|
49
|
+
/* ------------------------------------------------------------------ motion */
|
|
50
|
+
|
|
51
|
+
const W = 128;
|
|
52
|
+
const H = Math.max(2, Math.round((W * vs.height) / vs.width / 2) * 2);
|
|
53
|
+
const v = spawnSync(FFMPEG, ['-v', 'error', ...range, '-i', clip, '-vf', `scale=${W}:${H},format=gray`, '-f', 'rawvideo', '-'], {maxBuffer: 1 << 30});
|
|
54
|
+
const px = W * H;
|
|
55
|
+
const nFrames = Math.floor(v.stdout.length / px);
|
|
56
|
+
const frame = (i) => v.stdout.subarray(i * px, (i + 1) * px);
|
|
57
|
+
const diff = [0];
|
|
58
|
+
const hist = (buf) => {
|
|
59
|
+
const h = new Float64Array(16);
|
|
60
|
+
for (let k = 0; k < buf.length; k++) h[buf[k] >> 4]++;
|
|
61
|
+
return h.map((x) => x / buf.length);
|
|
62
|
+
};
|
|
63
|
+
const histDist = [0];
|
|
64
|
+
let prevH = hist(frame(0));
|
|
65
|
+
for (let i = 1; i < nFrames; i++) {
|
|
66
|
+
const a = frame(i - 1);
|
|
67
|
+
const b = frame(i);
|
|
68
|
+
let s = 0;
|
|
69
|
+
for (let k = 0; k < px; k++) s += Math.abs(a[k] - b[k]);
|
|
70
|
+
diff.push(s / px);
|
|
71
|
+
const hcur = hist(b);
|
|
72
|
+
let hd = 0;
|
|
73
|
+
for (let k = 0; k < 16; k++) hd += Math.abs(hcur[k] - prevH[k]);
|
|
74
|
+
histDist.push(hd / 2);
|
|
75
|
+
prevH = hcur;
|
|
76
|
+
}
|
|
77
|
+
const sorted = [...diff].sort((x, y) => x - y);
|
|
78
|
+
const pct = (p) => sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))];
|
|
79
|
+
const floor = Math.max(0.6, pct(0.25) * 2.5);
|
|
80
|
+
|
|
81
|
+
// A cut: one frame that changes far more than both neighbours, and changes the picture's tones too.
|
|
82
|
+
const cuts = [];
|
|
83
|
+
for (let i = 1; i < nFrames - 1; i++) {
|
|
84
|
+
const nb = Math.max(diff[i - 1], diff[i + 1], floor);
|
|
85
|
+
if (diff[i] > 3 * nb && diff[i] > 8 && histDist[i] > 0.12) cuts.push(i);
|
|
86
|
+
}
|
|
87
|
+
const isCut = new Set(cuts);
|
|
88
|
+
|
|
89
|
+
// Moves: runs of frames above the noise floor (cut frames excluded), smoothed over 3 frames.
|
|
90
|
+
const sm = diff.map((_, i) => (isCut.has(i) ? 0 : (diff[Math.max(0, i - 1)] * (isCut.has(i - 1) ? 0 : 1) + diff[i] + (isCut.has(i + 1) ? 0 : diff[Math.min(nFrames - 1, i + 1)])) / 3));
|
|
91
|
+
const moves = [];
|
|
92
|
+
for (let i = 0; i < nFrames; ) {
|
|
93
|
+
if (sm[i] <= floor) {
|
|
94
|
+
i++;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
let j = i;
|
|
98
|
+
while (j < nFrames && sm[j] > floor) j++;
|
|
99
|
+
if (j - i >= 3) {
|
|
100
|
+
let pk = i;
|
|
101
|
+
for (let k = i; k < j; k++) if (sm[k] > sm[pk]) pk = k;
|
|
102
|
+
moves.push({start: i, end: j - 1, peak: pk, frames: j - i, rise: pk - i, settle: j - 1 - pk, strength: sm[pk]});
|
|
103
|
+
}
|
|
104
|
+
i = j;
|
|
105
|
+
}
|
|
106
|
+
const holds = [];
|
|
107
|
+
const events = [...cuts.map((c) => [c, c]), ...moves.map((m) => [m.start, m.end])].sort((a, b) => a[0] - b[0]);
|
|
108
|
+
for (let k = 1; k < events.length; k++) {
|
|
109
|
+
const gap = events[k][0] - events[k - 1][1];
|
|
110
|
+
if (gap > 1) holds.push(gap);
|
|
111
|
+
}
|
|
112
|
+
const inMotion = sm.filter((x) => x > floor).length / nFrames;
|
|
113
|
+
const med = (a) => (a.length ? [...a].sort((x, y) => x - y)[Math.floor(a.length / 2)] : 0);
|
|
114
|
+
|
|
115
|
+
/* ------------------------------------------------------------------ sound */
|
|
116
|
+
|
|
117
|
+
const SR = 22050;
|
|
118
|
+
const a = spawnSync(FFMPEG, ['-v', 'error', ...range, '-i', clip, '-vn', '-ac', '1', '-ar', String(SR), '-f', 's16le', '-'], {maxBuffer: 1 << 30});
|
|
119
|
+
const hasAudio = a.stdout && a.stdout.length > SR;
|
|
120
|
+
let sound = {present: false};
|
|
121
|
+
if (hasAudio) {
|
|
122
|
+
const pcm = new Int16Array(a.stdout.buffer.slice(a.stdout.byteOffset, a.stdout.byteOffset + (a.stdout.length & ~1)));
|
|
123
|
+
const N = 1024;
|
|
124
|
+
const HOP = 512;
|
|
125
|
+
const win = Float64Array.from({length: N}, (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / N));
|
|
126
|
+
// A small radix-2 FFT, magnitude only — enough for flux and centroid.
|
|
127
|
+
const fft = (re) => {
|
|
128
|
+
const n = re.length;
|
|
129
|
+
const im = new Float64Array(n);
|
|
130
|
+
for (let i = 1, j = 0; i < n; i++) {
|
|
131
|
+
let bit = n >> 1;
|
|
132
|
+
for (; j & bit; bit >>= 1) j ^= bit;
|
|
133
|
+
j ^= bit;
|
|
134
|
+
if (i < j) [re[i], re[j]] = [re[j], re[i]];
|
|
135
|
+
}
|
|
136
|
+
for (let len = 2; len <= n; len <<= 1) {
|
|
137
|
+
const ang = (-2 * Math.PI) / len;
|
|
138
|
+
for (let i = 0; i < n; i += len)
|
|
139
|
+
for (let k = 0; k < len / 2; k++) {
|
|
140
|
+
const cr = Math.cos(ang * k);
|
|
141
|
+
const ci = Math.sin(ang * k);
|
|
142
|
+
const ur = re[i + k];
|
|
143
|
+
const ui = im[i + k];
|
|
144
|
+
const vr = re[i + k + len / 2] * cr - im[i + k + len / 2] * ci;
|
|
145
|
+
const vi = re[i + k + len / 2] * ci + im[i + k + len / 2] * cr;
|
|
146
|
+
re[i + k] = ur + vr;
|
|
147
|
+
im[i + k] = ui + vi;
|
|
148
|
+
re[i + k + len / 2] = ur - vr;
|
|
149
|
+
im[i + k + len / 2] = ui - vi;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
return Float64Array.from({length: n / 2}, (_, k) => Math.hypot(re[k], im[k]));
|
|
153
|
+
};
|
|
154
|
+
const rms = [];
|
|
155
|
+
const flux = [];
|
|
156
|
+
const centroid = [];
|
|
157
|
+
let prev = new Float64Array(N / 2);
|
|
158
|
+
for (let off = 0; off + N <= pcm.length; off += HOP) {
|
|
159
|
+
const buf = new Float64Array(N);
|
|
160
|
+
let e = 0;
|
|
161
|
+
for (let i = 0; i < N; i++) {
|
|
162
|
+
const s = pcm[off + i] / 32768;
|
|
163
|
+
e += s * s;
|
|
164
|
+
buf[i] = s * win[i];
|
|
165
|
+
}
|
|
166
|
+
rms.push(Math.sqrt(e / N));
|
|
167
|
+
const mag = fft(buf);
|
|
168
|
+
let f = 0;
|
|
169
|
+
let num = 0;
|
|
170
|
+
let den = 0;
|
|
171
|
+
for (let k = 1; k < N / 2; k++) {
|
|
172
|
+
f += Math.max(0, mag[k] - prev[k]);
|
|
173
|
+
num += k * mag[k];
|
|
174
|
+
den += mag[k];
|
|
175
|
+
}
|
|
176
|
+
flux.push(f);
|
|
177
|
+
centroid.push(den ? ((num / den) * SR) / N : 0);
|
|
178
|
+
prev = mag;
|
|
179
|
+
}
|
|
180
|
+
const hopS = HOP / SR;
|
|
181
|
+
const dB = (x) => (x > 0 ? 20 * Math.log10(x) : -99);
|
|
182
|
+
// Onsets: flux peaks well above the local median.
|
|
183
|
+
const onsets = [];
|
|
184
|
+
for (let i = 2; i < flux.length - 2; i++) {
|
|
185
|
+
const lo = Math.max(0, i - 20);
|
|
186
|
+
const local = [...flux.slice(lo, i + 20)].sort((x, y) => x - y);
|
|
187
|
+
const thr = local[Math.floor(local.length / 2)] * 2.2 + 0.5;
|
|
188
|
+
if (flux[i] > thr && flux[i] >= flux[i - 1] && flux[i] >= flux[i + 1] && rms[i + 1] > 0.004) {
|
|
189
|
+
if (onsets.length && i - onsets.at(-1).i < 4) continue;
|
|
190
|
+
// Shape: how long the sound lasts above (peak − 12 dB), and how it built up before the onset.
|
|
191
|
+
let pk = i;
|
|
192
|
+
for (let k = i; k < Math.min(rms.length, i + 6); k++) if (rms[k] > rms[pk]) pk = k;
|
|
193
|
+
let end = pk;
|
|
194
|
+
while (end < rms.length - 1 && dB(rms[end]) > dB(rms[pk]) - 12 && end - pk < 120) end++;
|
|
195
|
+
let pre = i;
|
|
196
|
+
while (pre > 0 && rms[pre - 1] < rms[pre] && i - pre < 40) pre--;
|
|
197
|
+
const dur = (end - i) * hopS;
|
|
198
|
+
const build = (i - pre) * hopS;
|
|
199
|
+
const c = centroid[pk];
|
|
200
|
+
const kind = dur < 0.12 && c > 2500 ? 'click' : build > 0.15 && c > 1500 ? 'whoosh' : c < 1800 && dur >= 0.12 ? 'hit' : dur < 0.18 ? 'tick' : 'other';
|
|
201
|
+
// jump: how far the sound rises above what was playing just before it — a sound effect stands out
|
|
202
|
+
// from the voice/music bed; a syllable or a note mostly does not.
|
|
203
|
+
const bed = Math.max(...rms.slice(Math.max(0, i - 12), Math.max(1, i - 2)));
|
|
204
|
+
onsets.push({i, t: i * hopS, db: Math.round(dB(rms[pk])), jump: Math.round(dB(rms[pk]) - dB(bed)), flux: flux[i], centroid: Math.round(c), dur: Math.round(dur * 1000), kind});
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
// Tempo: autocorrelation of the onset strength over 60–180 BPM.
|
|
208
|
+
let bpm = null;
|
|
209
|
+
let pulse = 0;
|
|
210
|
+
const env = flux.map((x, i) => x - (flux[i - 1] ?? x));
|
|
211
|
+
const meanEnv = env.reduce((s, x) => s + Math.max(0, x), 0) / env.length;
|
|
212
|
+
let best = 0;
|
|
213
|
+
let bestLag = 0;
|
|
214
|
+
let base = 0;
|
|
215
|
+
for (let lag = Math.round(60 / 180 / hopS); lag <= Math.round(60 / 60 / hopS); lag++) {
|
|
216
|
+
let s = 0;
|
|
217
|
+
for (let i = lag; i < env.length; i++) s += Math.max(0, env[i]) * Math.max(0, env[i - lag]);
|
|
218
|
+
s /= env.length - lag;
|
|
219
|
+
base += s;
|
|
220
|
+
if (s > best) [best, bestLag] = [s, lag];
|
|
221
|
+
}
|
|
222
|
+
base /= Math.round(60 / 60 / hopS) - Math.round(60 / 180 / hopS) + 1;
|
|
223
|
+
pulse = base ? best / base : 0;
|
|
224
|
+
if (pulse > 1.6 && meanEnv > 0) bpm = Math.round(60 / (bestLag * hopS));
|
|
225
|
+
const lufs = spawnSync(FFMPEG, ['-v', 'info', ...range, '-i', clip, '-vn', '-af', 'ebur128=framelog=quiet', '-f', 'null', '-'], {encoding: 'utf8'}).stderr;
|
|
226
|
+
// Strong onsets: the ones that stand out — top 12% by onset strength AND at least 6 dB over the bed.
|
|
227
|
+
// These are the sound design (whooshes, hits, risers, music accents); the rest is mostly voice and music.
|
|
228
|
+
const fluxes = onsets.map((o) => o.flux).sort((x, y) => x - y);
|
|
229
|
+
const strongCut = fluxes[Math.floor(fluxes.length * 0.88)] ?? Infinity;
|
|
230
|
+
for (const o of onsets) o.strong = o.flux >= strongCut && o.jump >= 6;
|
|
231
|
+
sound = {
|
|
232
|
+
present: true,
|
|
233
|
+
lufs: Number(/I:\s+(-?[\d.]+) LUFS/.exec(lufs)?.[1] ?? NaN),
|
|
234
|
+
lra: Number(/LRA:\s+([\d.]+) LU/.exec(lufs)?.[1] ?? NaN),
|
|
235
|
+
bpm,
|
|
236
|
+
pulse: Math.round(pulse * 100) / 100,
|
|
237
|
+
onsets,
|
|
238
|
+
perMinute: Math.round((onsets.length / ((to - from) / 60)) * 10) / 10,
|
|
239
|
+
kinds: onsets.reduce((m, o) => ({...m, [o.kind]: (m[o.kind] ?? 0) + 1}), {}),
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* The sound as a picture, for whoever cannot listen: time left to right, pitch bottom to top, loudness as
|
|
245
|
+
* brightness. Risers climb diagonally, whooshes are short sweeps high up, hits are vertical lines, music is
|
|
246
|
+
* horizontal bands, a drop-out is a dark gap, a beat is a row of evenly spaced lines. One image per minute.
|
|
247
|
+
*/
|
|
248
|
+
const spectra = [];
|
|
249
|
+
if (hasAudio) {
|
|
250
|
+
for (let s = from; s < to && spectra.length < 5; s += 60) {
|
|
251
|
+
const file = join(out, `spectrum-${Math.round(s)}.png`);
|
|
252
|
+
spawnSync(FFMPEG, ['-y', '-v', 'error', '-ss', String(s), '-t', String(Math.min(60, to - s)), '-i', clip, '-lavfi', 'showspectrumpic=s=1400x360:legend=1:scale=log:fscale=log:color=intensity:gain=2', file]);
|
|
253
|
+
spectra.push(file);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/* ------------------------------------------------------------------ sync */
|
|
258
|
+
|
|
259
|
+
const nearest = (t) => {
|
|
260
|
+
const strong = sound.present ? sound.onsets.filter((o) => o.strong) : [];
|
|
261
|
+
if (!strong.length) return null;
|
|
262
|
+
let best = strong[0];
|
|
263
|
+
for (const o of strong) if (Math.abs(o.t - t) < Math.abs(best.t - t)) best = o;
|
|
264
|
+
const lead = Math.round((best.t - t) * fps * 10) / 10; // frames; negative = sound before picture
|
|
265
|
+
return Math.abs(lead) <= fps * 0.5 ? {lead, kind: best.kind} : null;
|
|
266
|
+
};
|
|
267
|
+
const cutSync = cuts.map((c) => nearest(c / fps)).filter(Boolean);
|
|
268
|
+
const moveSync = moves.map((m) => nearest(m.start / fps)).filter(Boolean);
|
|
269
|
+
|
|
270
|
+
/* ------------------------------------------------------------------ strips */
|
|
271
|
+
|
|
272
|
+
const strip = (startF, endF, label) => {
|
|
273
|
+
const s = Math.max(0, startF);
|
|
274
|
+
const e = Math.min(nFrames - 1, endF);
|
|
275
|
+
const n = e - s + 1;
|
|
276
|
+
const step = Math.max(1, Math.ceil(n / 12));
|
|
277
|
+
const count = Math.ceil(n / step);
|
|
278
|
+
const file = join(out, 'strips', `${label}.jpg`);
|
|
279
|
+
const tw = vs.width >= vs.height ? 320 : 180;
|
|
280
|
+
spawnSync(FFMPEG, ['-y', '-v', 'error', '-ss', String(from + s / fps), '-i', clip, '-frames:v', String(count), '-vf', `select='not(mod(n\\,${step}))',scale=${tw}:-2,tile=${Math.min(count, 6)}x${Math.ceil(count / 6)}:padding=4:color=white`, '-q:v', '4', file]);
|
|
281
|
+
return file;
|
|
282
|
+
};
|
|
283
|
+
const pickedMoves = [...moves].sort((x, y) => y.strength * y.frames - x.strength * x.frames).slice(0, maxStrips);
|
|
284
|
+
const strips = [];
|
|
285
|
+
for (const c of cuts.slice(0, maxStrips)) strips.push({what: 'cut', at: tc(from + c / fps), file: strip(c - 4, c + 7, `cut-${(from + c / fps).toFixed(2)}`)});
|
|
286
|
+
for (const m of pickedMoves) strips.push({what: `move ${m.frames}f`, at: tc(from + m.start / fps), file: strip(m.start - 1, m.end + 1, `move-${(from + m.start / fps).toFixed(2)}`)});
|
|
287
|
+
|
|
288
|
+
/* ------------------------------------------------------------------ report */
|
|
289
|
+
|
|
290
|
+
const easeOf = (m) => (m.settle >= m.rise * 1.6 ? 'ease-out (fast start, long settle)' : m.rise >= m.settle * 1.6 ? 'ease-in (slow start, sharp stop)' : 'ease-in-out');
|
|
291
|
+
const report = {
|
|
292
|
+
clip: basename(clip),
|
|
293
|
+
window: [from, to],
|
|
294
|
+
fps,
|
|
295
|
+
size: `${vs.width}x${vs.height}`,
|
|
296
|
+
motion: {
|
|
297
|
+
frames: nFrames,
|
|
298
|
+
cuts: cuts.length,
|
|
299
|
+
cutsPerMinute: Math.round((cuts.length / ((to - from) / 60)) * 10) / 10,
|
|
300
|
+
moves: moves.length,
|
|
301
|
+
inMotionShare: Math.round(inMotion * 100),
|
|
302
|
+
medianMoveFrames: med(moves.map((m) => m.frames)),
|
|
303
|
+
medianRise: med(moves.map((m) => m.rise)),
|
|
304
|
+
medianSettle: med(moves.map((m) => m.settle)),
|
|
305
|
+
medianHoldFrames: med(holds),
|
|
306
|
+
easing: moves.reduce((acc, m) => ({...acc, [easeOf(m)]: (acc[easeOf(m)] ?? 0) + 1}), {}),
|
|
307
|
+
},
|
|
308
|
+
sound,
|
|
309
|
+
sync: {
|
|
310
|
+
cutsWithSound: `${cutSync.length}/${cuts.length}`,
|
|
311
|
+
medianCutLeadFrames: med(cutSync.map((s) => s.lead)),
|
|
312
|
+
movesWithSound: `${moveSync.length}/${moves.length}`,
|
|
313
|
+
medianMoveLeadFrames: med(moveSync.map((s) => s.lead)),
|
|
314
|
+
},
|
|
315
|
+
cutTimes: cuts.map((c) => tc(from + c / fps)),
|
|
316
|
+
moveList: moves.map((m) => ({at: tc(from + m.start / fps), frames: m.frames, rise: m.rise, settle: m.settle, ease: easeOf(m), strength: Math.round(m.strength * 10) / 10})),
|
|
317
|
+
strips,
|
|
318
|
+
};
|
|
319
|
+
if (sound.present) report.sound = {...sound, onsets: sound.onsets.map((o) => ({...o, at: tc(from + o.t)}))};
|
|
320
|
+
writeFileSync(join(out, 'report.json'), JSON.stringify(report, null, 1));
|
|
321
|
+
|
|
322
|
+
const M = report.motion;
|
|
323
|
+
const md = [
|
|
324
|
+
`# ${report.clip} — ${tc(from)}–${tc(to)} · ${report.size} @ ${fps.toFixed(2)}fps`,
|
|
325
|
+
'',
|
|
326
|
+
`**Motion.** ${M.cuts} hard cuts (${M.cutsPerMinute}/min) · ${M.moves} moves · in motion ${M.inMotionShare}% of frames · median move ${M.medianMoveFrames}f (${(M.medianMoveFrames / fps).toFixed(2)}s), reaching full speed in ${M.medianRise}f and settling over ${M.medianSettle}f · median hold ${M.medianHoldFrames}f (${(M.medianHoldFrames / fps).toFixed(2)}s).`,
|
|
327
|
+
`Easing: ${Object.entries(M.easing)
|
|
328
|
+
.map(([k, n]) => `${n}× ${k}`)
|
|
329
|
+
.join(', ')}.`,
|
|
330
|
+
'',
|
|
331
|
+
sound.present
|
|
332
|
+
? `**Sound.** ${report.sound.onsets.filter((o) => o.strong).length} standout hits (${(report.sound.onsets.filter((o) => o.strong).length / ((to - from) / 60)).toFixed(1)}/min: ${Object.entries(report.sound.onsets.filter((o) => o.strong).reduce((m, o) => ({...m, [o.kind]: (m[o.kind] ?? 0) + 1}), {})).map(([k, n]) => `${n} ${k}`).join(', ')}) · ${report.sound.lufs} LUFS (range ${report.sound.lra} LU) · ${report.sound.onsets.length} onsets (${report.sound.perMinute}/min): ${Object.entries(report.sound.kinds)
|
|
333
|
+
.map(([k, n]) => `${n} ${k}`)
|
|
334
|
+
.join(', ')} · ${report.sound.bpm ? `steady pulse ~${report.sound.bpm} BPM (strength ${report.sound.pulse})` : `no steady beat (pulse ${report.sound.pulse})`}.`
|
|
335
|
+
: '**Sound.** none in this file.',
|
|
336
|
+
`**Sync** (standout hits only). cuts with a hit within ½s: ${report.sync.cutsWithSound} (median ${report.sync.medianCutLeadFrames}f, − = sound first) · moves: ${report.sync.movesWithSound} (median ${report.sync.medianMoveLeadFrames}f).`,
|
|
337
|
+
'',
|
|
338
|
+
...(spectra.length ? ['## The sound, as a picture (risers climb, whooshes sweep high, hits are vertical lines, music is bands, a drop is a dark gap)', ...spectra.map((f) => `- ${f.replace(/\\/g, '/')}`), ''] : []),
|
|
339
|
+
'## Strips (every frame across the event)',
|
|
340
|
+
...strips.map((s) => `- ${s.what} at ${s.at}: ${s.file.replace(/\\/g, '/')}`),
|
|
341
|
+
'',
|
|
342
|
+
'## Moves',
|
|
343
|
+
...report.moveList.slice(0, 40).map((m) => `- ${m.at} ${m.frames}f rise ${m.rise} / settle ${m.settle} ${m.ease} (strength ${m.strength})`),
|
|
344
|
+
'',
|
|
345
|
+
'## Cuts',
|
|
346
|
+
report.cutTimes.join(' · ') || 'none',
|
|
347
|
+
].join('\n');
|
|
348
|
+
writeFileSync(join(out, 'report.md'), md);
|
|
349
|
+
console.log(md.split('\n').slice(0, 7).join('\n'));
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"""Download, once and up front, every model the studio will need — for the language its videos are in.
|
|
2
|
+
|
|
3
|
+
Usage (from the studio folder, with the studio's Python):
|
|
4
|
+
python scripts/setup_models.py --language ml # en hi hinglish te ml ur ta kn mr bn gu pa, or any code
|
|
5
|
+
python scripts/setup_models.py --language en --no-tracking
|
|
6
|
+
|
|
7
|
+
Saves the choice to studio.config.json, then fetches into the studio's .models folder:
|
|
8
|
+
1. the transcriber for that language (English: small, ~0.5 GB; Indian languages: large-v3, ~3 GB),
|
|
9
|
+
2. its word-timing model (~0.4–2.5 GB),
|
|
10
|
+
3. motion tracking: the person-finding model (~0.2 GB) and a check that the tracker loads.
|
|
11
|
+
|
|
12
|
+
Why up front: these downloads are large, and finding that out in the middle of someone's first edit — with
|
|
13
|
+
their footage waiting — is the worst moment to learn it (Malayalam needed ~5 GB the first time).
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import argparse
|
|
17
|
+
import json
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
import time
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
|
|
23
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
24
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent / "lib"))
|
|
25
|
+
from media import FFMPEG, use_certifi # noqa: E402
|
|
26
|
+
from languages import CONFIG, LANGS, lang, load_aligner # noqa: E402
|
|
27
|
+
|
|
28
|
+
ROOT = Path(__file__).resolve().parent.parent
|
|
29
|
+
MODELS = ROOT / ".models"
|
|
30
|
+
os.environ.setdefault("HF_HOME", str(MODELS / "huggingface"))
|
|
31
|
+
os.environ.setdefault("TORCH_HOME", str(MODELS / "torch"))
|
|
32
|
+
os.environ["PATH"] = str(Path(FFMPEG).parent) + os.pathsep + os.environ["PATH"]
|
|
33
|
+
use_certifi()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def size_gb(path: Path) -> float:
|
|
37
|
+
return sum(f.stat().st_size for f in path.rglob("*") if f.is_file()) / 1e9 if path.exists() else 0.0
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def main() -> None:
|
|
41
|
+
for stream in (sys.stdout, sys.stderr):
|
|
42
|
+
try:
|
|
43
|
+
stream.reconfigure(encoding="utf-8", errors="replace")
|
|
44
|
+
except (AttributeError, ValueError):
|
|
45
|
+
pass
|
|
46
|
+
ap = argparse.ArgumentParser()
|
|
47
|
+
ap.add_argument("--language", default="en", help=f"one of {', '.join(LANGS)} — or any language code")
|
|
48
|
+
ap.add_argument("--no-tracking", action="store_true", help="skip the motion-tracking model")
|
|
49
|
+
args = ap.parse_args()
|
|
50
|
+
|
|
51
|
+
choice = args.language.strip().lower()
|
|
52
|
+
info = lang(choice)
|
|
53
|
+
(ROOT / CONFIG).write_text(json.dumps({"language": choice}, indent=2) + "\n", encoding="utf-8")
|
|
54
|
+
print(f"Language: {info['name']} — saved to {CONFIG}")
|
|
55
|
+
started = time.time()
|
|
56
|
+
before = size_gb(MODELS)
|
|
57
|
+
ok = True
|
|
58
|
+
|
|
59
|
+
import whisperx # noqa: E402 (after the cache/cert setup above)
|
|
60
|
+
|
|
61
|
+
print(f"1/3 Transcriber ({info['whisper']})…", flush=True)
|
|
62
|
+
whisperx.load_model(info["whisper"], device="cpu", compute_type="int8", language=info["code"])
|
|
63
|
+
print(" ready")
|
|
64
|
+
|
|
65
|
+
print("2/3 Word timing…", flush=True)
|
|
66
|
+
model, _ = load_aligner(whisperx, info["code"])
|
|
67
|
+
if model is None:
|
|
68
|
+
print(" none for this language — captions will use the transcriber's own timings (looser).")
|
|
69
|
+
else:
|
|
70
|
+
print(" ready" + ("" if info["tested"] else " (experimental for this language — check the first video's captions)"))
|
|
71
|
+
|
|
72
|
+
if not args.no_tracking:
|
|
73
|
+
print("3/3 Motion tracking…", flush=True)
|
|
74
|
+
try:
|
|
75
|
+
import cv2 # noqa: F401
|
|
76
|
+
from make_cutout import load_segmenter
|
|
77
|
+
|
|
78
|
+
load_segmenter("lite")
|
|
79
|
+
print(" ready")
|
|
80
|
+
except Exception as exc: # noqa: BLE001
|
|
81
|
+
ok = False
|
|
82
|
+
print(f" FAILED: {exc}. Run: pip install -r requirements.txt with the studio's Python, then this again.")
|
|
83
|
+
|
|
84
|
+
print(f"Done in {time.time() - started:.0f}s — {size_gb(MODELS) - before:.1f} GB downloaded, {size_gb(MODELS):.1f} GB of models in .models")
|
|
85
|
+
raise SystemExit(0 if ok else 1)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
if __name__ == "__main__":
|
|
89
|
+
main()
|
|
@@ -31,7 +31,7 @@ const out = opt('out') ?? join('work', 'stills', id);
|
|
|
31
31
|
rmSync(out, {recursive: true, force: true});
|
|
32
32
|
mkdirSync(out, {recursive: true});
|
|
33
33
|
|
|
34
|
-
const serveUrl = await bundle({entryPoint: resolve('src/index.ts'), publicDir: resolve('assets')});
|
|
34
|
+
const serveUrl = await bundle({entryPoint: resolve(opt('entry') ?? 'src/index.ts'), publicDir: resolve(opt('public') ?? 'assets')});
|
|
35
35
|
const comp = (await getCompositions(serveUrl)).find((c) => c.id === id);
|
|
36
36
|
if (!comp) throw new Error(`No composition ${id}`);
|
|
37
37
|
|