creator-editing-studio 1.3.0 → 1.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "creator-editing-studio",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "Scaffolds a Creator Editing OS studio — components, scripts and rules for editing video from a plain-English brief.",
   "license": "UNLICENSED",
   "type": "module",

package/template/CLAUDE.md

@@ -167,10 +167,33 @@ and not in the stills, it did not render — find out why before shipping.
 `OffthreadVideo` placed in one starts from its own frame 0 and plays its own audio over the
 live line. Pass an explicit `startFrom`, and mute it unless its sound is wanted.
 
-**Repetition the owner can hear but you cannot.** If the owner says a line repeats, transcribe
-the rendered output over that exact window before arguing. If the audio is clean, the repeat
-is visual — a held caption, a reused graphic, or a duplicated framing — and that is still a
-real fault worth fixing, not a misunderstanding to explain away.
+**Something they can hear and you cannot.** When they report a fault you cannot find, your check
+is the thing most likely to be wrong, not them. Three rules, learned the hard way:
+
+- **Transcribe the WHOLE file, not the window they named.** A seven-second sample came back clean
+  and was used to tell the owner he was imagining a repeat that was really there.
+- **Check every render in `out/`, not the one you just made.** They are probably playing an older
+  file, because nothing on their machine says which is current. Ask which file, or test them all.
+- **If they say it more than once, stop checking and start believing.** A repeated complaint is
+  evidence your method is wrong. Change the method.
+
+If the audio really is clean, the repeat is visual — a held caption, a reused graphic, a duplicated
+framing — and that is still a real fault, not a misunderstanding to explain away.
+
+**A transcript is not evidence.** Whisper suppresses repeated phrases: when someone says a line
+twice it transcribes it once and stretches the last recognised word across the speech it dropped,
+so the transcript reads clean while the audio plainly repeats. Run `node scripts/coverage.mjs
+<media> <words.json>` after every transcription and before planning anything — it flags any word
+far longer than the median, and any stretch above the noise floor with no word on it. Both mean
+the transcript is hiding speech. Check the energy across the flagged word: one decaying word is
+fine, a dip to silence followed by a second loud onset is a repeat.
+
+**Four generations of lossy audio.** The voice must be cut as PCM and encoded exactly once, at
+delivery. Cutting an AAC file, trimming the result, and then letting the renderer re-encode it
+puts the voice through four generations of loss, and raising the bitrate on the later passes
+preserves the damage rather than repairing it. It is clearly audible and it gets worse with
+every extra render. Use `node scripts/voice.mjs` to pull the voice losslessly from the camera
+file, render the picture muted, and mux the two together at the end.
 
 ## Non-negotiables
 

package/template/scripts/coverage.mjs

@@ -0,0 +1,135 @@
+#!/usr/bin/env node
+/**
+ * Find speech the transcriber did not transcribe.
+ *
+ *   node scripts/coverage.mjs <media> <words.json> [--gap 0.35] [--floor -38]
+ *
+ * WHY THIS EXISTS. Whisper suppresses repeated phrases. When a speaker says the same line twice,
+ * it transcribes it once and silently leaves a hole where the duplicate was — so a transcript can
+ * read perfectly clean while the audio plainly repeats. On demo-test that cost four rounds: the
+ * owner said three times that "adds a few zooms" repeated, the transcript said it did not, and the
+ * hole got mistaken for a pause and trimmed.
+ *
+ * A transcript is not evidence on its own. This checks it against the audio: any stretch that is
+ * loud enough to be speech but carries no word is either a dropped duplicate or a dropped line,
+ * and both are faults. Run it after every transcription, before planning anything.
+ */
+import {spawnSync} from 'node:child_process';
+import {readFileSync} from 'node:fs';
+import {FFMPEG} from './lib/media.mjs';
+
+const [, , media, wordsFile, ...rest] = process.argv;
+if (!media || !wordsFile) {
+  console.error('Usage: node scripts/coverage.mjs <media> <words.json> [--gap 0.35] [--floor -38]');
+  process.exit(2);
+}
+const opt = (n, d) => {
+  const i = rest.indexOf(`--${n}`);
+  return i >= 0 ? Number(rest[i + 1]) : d;
+};
+/** Shortest unlabelled speech run worth reporting, in seconds. */
+const MIN_GAP = opt('gap', 0.35);
+/** Above this RMS it is speech, below it is room tone. */
+const FLOOR_DB = opt('floor', -38);
+
+const SR = 16000;
+const WIN = Math.round(SR * 0.05);
+
+const pcm = spawnSync(FFMPEG, ['-v', 'error', '-i', media, '-vn', '-ac', '1', '-ar', String(SR), '-f', 's16le', '-'], {
+  encoding: 'buffer',
+  maxBuffer: 1 << 28,
+});
+if (pcm.status !== 0) {
+  console.error(pcm.stderr?.toString().slice(0, 600));
+  process.exit(1);
+}
+const buf = pcm.stdout;
+const samples = buf.length / 2;
+
+const loud = (t) => {
+  const a = Math.round(t * SR);
+  let s = 0;
+  let c = 0;
+  for (let i = a; i < a + WIN && i < samples; i++) {
+    const v = buf.readInt16LE(i * 2);
+    s += v * v;
+    c++;
+  }
+  const r = Math.sqrt(s / Math.max(1, c));
+  return r > 0 ? 20 * Math.log10(r / 32768) : -99;
+};
+
+const data = JSON.parse(readFileSync(wordsFile, 'utf8'));
+const words = data.words ?? data;
+const fps = data.fps ?? 30;
+/** Words may carry frames (f/t) or seconds (start/end). */
+const spans = words.map((w) => ({
+  from: w.start ?? w.f / fps,
+  to: w.end ?? (w.t ?? w.f) / fps,
+}));
+
+const covered = (t) => spans.some((s) => t >= s.from - 0.06 && t <= s.to + 0.06);
+
+const duration = samples / SR;
+const holes = [];
+let run = null;
+for (let t = 0; t < duration; t += 0.05) {
+  const speech = loud(t) > FLOOR_DB;
+  if (speech && !covered(t)) {
+    if (!run) run = {from: t, to: t};
+    else run.to = t;
+  } else if (run) {
+    if (run.to - run.from >= MIN_GAP) holes.push(run);
+    run = null;
+  }
+}
+if (run && run.to - run.from >= MIN_GAP) holes.push(run);
+
+/**
+ * The detector that actually works.
+ *
+ * A collapsed duplicate does not leave an untranscribed hole — Whisper stretches the LAST word it
+ * did recognise across the speech it dropped. On demo-test the word "adds" came back 0.68s long
+ * with the lowest confidence in the sentence, covering a whole repeated phrase, so a coverage test
+ * based on spans called the file clean. Word duration is what gives it away.
+ */
+const lengths = words.map((w, i) => ({w, i, dur: spans[i].to - spans[i].from})).filter((x) => x.dur > 0);
+const sortedDur = lengths.map((x) => x.dur).sort((a, b) => a - b);
+const median = sortedDur[Math.floor(sortedDur.length / 2)] || 0.2;
+const stretched = lengths.filter((x) => x.dur > median * 3);
+if (stretched.length) {
+  console.log(
+    `${stretched.length} word(s) far longer than the ${median.toFixed(2)}s median — ` +
+      `each may be covering speech the transcriber dropped:\n`,
+  );
+  for (const x of stretched) {
+    const label = x.w.w ?? x.w.word;
+    console.log(
+      `  "${label}" ${spans[x.i].from.toFixed(2)}s - ${spans[x.i].to.toFixed(2)}s = ` +
+        `${x.dur.toFixed(2)}s (${(x.dur / median).toFixed(1)}x median)`,
+    );
+  }
+  console.log('\nCheck the energy across each one. A single decaying word is fine; a dip to');
+  console.log('silence followed by a second loud onset is a repeat the transcript is hiding.');
+}
+
+if (!holes.length && !stretched.length) {
+  console.log(`clean — every stretch above ${FLOOR_DB}dB has a word on it (${words.length} words, ${duration.toFixed(2)}s)`);
+  process.exit(0);
+}
+
+if (!holes.length) process.exit(1);
+
+console.log(`${holes.length} stretch(es) of SPEECH WITH NO WORDS — the transcript is missing something:\n`);
+for (const h of holes) {
+  const before = spans.filter((s) => s.to <= h.from).pop();
+  const after = spans.find((s) => s.from >= h.to);
+  const bw = before ? words[spans.indexOf(before)] : null;
+  const aw = after ? words[spans.indexOf(after)] : null;
+  console.log(
+    `  ${h.from.toFixed(2)}s - ${h.to.toFixed(2)}s  (${(h.to - h.from).toFixed(2)}s)` +
+      `  between "${bw ? bw.w ?? bw.word : 'start'}" and "${aw ? aw.w ?? aw.word : 'end'}"`,
+  );
+}
+console.log('\nMost likely a repeated phrase the transcriber collapsed. Listen to it before cutting it.');
+process.exit(1);

package/template/scripts/exact-frames.mjs

@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+/**
+ * Pull EXACT frame numbers out of a rendered file.
+ *
+ *   node scripts/exact-frames.mjs <file.mp4> <frame,frame,...> <outDir>
+ *
+ * Why this exists: `-ss` seeking is not frame accurate. Checking v4's cut boundaries with it
+ * returned frames 1-3 later than asked for, which made clean cuts look like defects and nearly
+ * sent a good edit back for rework. Decoding from the start and selecting by frame index is
+ * slower and is the only thing that can be trusted for a boundary check.
+ *
+ * The select expression is passed as one argv entry through spawnSync, never a shell — its
+ * commas are exactly what breaks a shell-quoted filter graph.
+ */
+import {spawnSync} from 'node:child_process';
+import {mkdirSync, rmSync} from 'node:fs';
+import {join} from 'node:path';
+import {FFMPEG} from './lib/media.mjs';
+
+const [, , file, framesArg, outDir = 'work/exact'] = process.argv;
+if (!file || !framesArg) {
+  console.error('Usage: node scripts/exact-frames.mjs <file> <frame,frame,...> [outDir]');
+  process.exit(2);
+}
+
+const frames = framesArg.split(',').map(Number).sort((a, b) => a - b);
+rmSync(outDir, {recursive: true, force: true});
+mkdirSync(outDir, {recursive: true});
+
+// eq(n\,N) — the backslash has to survive into the filter string, so it is doubled here.
+const expr = frames.map((f) => `eq(n\\,${f})`).join('+');
+
+const r = spawnSync(
+  FFMPEG,
+  [
+    '-y', '-v', 'error',
+    '-i', file,
+    '-vf', `select='${expr}',scale=440:-1`,
+    '-fps_mode', 'passthrough',
+    '-frames:v', String(frames.length),
+    join(outDir, 'x%03d.png'),
+  ],
+  {encoding: 'utf8'},
+);
+if (r.status !== 0) {
+  console.error(r.stderr?.slice(0, 900));
+  process.exit(1);
+}
+
+console.log(frames.map((f, i) => `x${String(i + 1).padStart(3, '0')}.png = frame ${f}`).join('\n'));

package/template/scripts/voice.mjs

@@ -0,0 +1,88 @@
+#!/usr/bin/env node
+/**
+ * Build the voice track ONCE, losslessly, straight from the camera file.
+ *
+ *   node scripts/voice.mjs --source <clip.mp4> --takes <takes.json> [--drop 17.2333:17.6667] --out <voice.wav>
+ *
+ * WHY THIS EXISTS. The voice on demo-test went through FOUR lossy generations before it reached
+ * the viewer: the camera's own AAC 128k, then the cut, then a second trim, then the render — each
+ * one re-encoding what the last one had already damaged. Raising the bitrate on the later passes
+ * does not recover anything; it just preserves the damage at a bigger file size. It is audible,
+ * and it gets worse with every extra render.
+ *
+ * The rulebook already says it: render the picture muted and add the original voice back
+ * untouched. This is the tool that makes that easy. The voice is cut as PCM and stays PCM until
+ * the single encode at delivery, so the viewer hears one generation of loss instead of four.
+ *
+ * `atrim` takes colon-separated options, so the filter graph carries no commas inside a function
+ * call — and it is passed through spawnSync as one argv entry, never a shell.
+ */
+import {spawnSync} from 'node:child_process';
+import {readFileSync} from 'node:fs';
+import {FFMPEG} from './lib/media.mjs';
+
+const args = process.argv.slice(2);
+const opt = (name) => {
+  const i = args.indexOf(`--${name}`);
+  return i >= 0 ? args[i + 1] : undefined;
+};
+
+const source = opt('source');
+const takesFile = opt('takes');
+const out = opt('out');
+if (!source || !out) {
+  console.error('Usage: node scripts/voice.mjs --source <clip> [--takes <takes.json>] [--drop a:b] --out <voice.wav>');
+  process.exit(2);
+}
+
+/** The spans to keep, in source seconds. With no takes file, keep the whole clip. */
+let keep = [{from: 0, to: null}];
+if (takesFile) keep = JSON.parse(readFileSync(takesFile, 'utf8')).keep;
+
+const parts = [];
+keep.forEach((k, i) => {
+  const range = k.to == null ? `start=${k.from.toFixed(4)}` : `start=${k.from.toFixed(4)}:end=${k.to.toFixed(4)}`;
+  parts.push(`[0:a]atrim=${range},asetpts=PTS-STARTPTS[a${i}]`);
+});
+parts.push(`${keep.map((_, i) => `[a${i}]`).join('')}concat=n=${keep.length}:v=0:a=1[joined]`);
+
+/**
+ * A second pass of removals, expressed in the CUT timeline rather than the source — that is where
+ * a tightened pause lives, because its position only exists after the first cut.
+ */
+const drop = opt('drop');
+let label = 'joined';
+if (drop) {
+  const [a, b] = drop.split(':').map(Number);
+  parts.push(`[joined]asplit=2[d0][d1]`);
+  parts.push(`[d0]atrim=start=0:end=${a.toFixed(4)},asetpts=PTS-STARTPTS[p0]`);
+  parts.push(`[d1]atrim=start=${b.toFixed(4)},asetpts=PTS-STARTPTS[p1]`);
+  parts.push(`[p0][p1]concat=n=2:v=0:a=1[cut]`);
+  label = 'cut';
+}
+
+const r = spawnSync(
+  FFMPEG,
+  [
+    '-y', '-v', 'error',
+    '-i', source,
+    '-filter_complex', parts.join(';'),
+    '-map', `[${label}]`,
+    // PCM, and the source's own sample rate — no resampling, no encoding.
+    '-c:a', 'pcm_s16le',
+    out,
+  ],
+  {encoding: 'utf8'},
+);
+if (r.status !== 0) {
+  console.error(r.stderr?.slice(0, 900));
+  process.exit(1);
+}
+
+const probe = spawnSync(
+  FFMPEG.replace(/ffmpeg(\.exe)?$/i, (m) => (m.endsWith('.exe') ? 'ffprobe.exe' : 'ffprobe')),
+  ['-v', 'error', '-show_entries', 'format=duration', '-of', 'csv=p=0', out],
+  {encoding: 'utf8'},
+);
+console.log(`voice written lossless -> ${out}  (${Number(probe.stdout).toFixed(3)}s, pcm_s16le)`);
+console.log('Render the picture muted, then mux this in with ONE encode at delivery.');