@aicopycoders/exodus 2026.9.2000-beta.8 → 2026.9.2000

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,10 @@
2
2
  // A plain first cut from a folder written by `exodus video pull`: the A-roll
3
3
  // scenes in the manifest's order make the spine, each cutaway lays over the
4
4
  // spine at the moment its cued line is spoken, a room-tone bed and the music
5
- // bed go underneath, and one MP4 comes out ready to upload.
5
+ // bed go underneath, and one MP4 comes out ready to upload. A reaction
6
+ // cutaway is the exception: it is its own short beat in the spine right after
7
+ // its cued line ends, heard with its own sound while the speaker is silent,
8
+ // fading in across any quiet start and out rather than stopping dead.
6
9
  //
7
10
  // node first-cut.mjs <pulled-dir> [--out cut.mp4] [--skip 2,5] [--no-music]
8
11
  //
@@ -11,9 +14,15 @@
11
14
  // recorded: the voice replaces the clip's audio, and the picture is fitted to
12
15
  // the voice, trimmed when the clip runs longer and held on its last frame when
13
16
  // the voice runs longer. The voice itself is never altered. A dialogue scene
14
- // keeps the audio it performed. A scene with no clip but a keyframe becomes a
15
- // still for the length of its voice track, else the storyboard's planned
16
- // duration. Every segment is loudness-normalized to -16 LUFS before the join.
17
+ // keeps the audio it performed; at each join its silence before the first word
18
+ // and after the last is cut back to a short beat, read from its word times.
19
+ // Inside it, a silence over 0.9 s between two sentences is cut down to 0.8 s,
20
+ // picture and sound together, unless the script writes that pause in. A
21
+ // sound in such a break that holds no heard word (a burp, a click) is cut
22
+ // out, read from the clip's loudness, and the same rule then applies. A
23
+ // scene with no clip but a keyframe becomes a still for the length of its voice
24
+ // track, else the storyboard's planned duration. Every segment is
25
+ // loudness-normalized to -16 LUFS before the join.
17
26
  // --skip drops scenes.
18
27
  //
19
28
  // After the render it writes the whole ad's word times beside the MP4, as
@@ -30,15 +39,54 @@ const FPS = 24;
30
39
  const norm = (label) =>
31
40
  `[${label}]scale=${WIDTH}:${HEIGHT}:force_original_aspect_ratio=decrease,` +
32
41
  `pad=${WIDTH}:${HEIGHT}:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=${FPS},format=yuv420p`;
42
+ /**
43
+ * Edges the cut chooses (join trims, reaction splits, narration lengths) land
44
+ * on whole frames, so the picture runs as long as its sound there and the
45
+ * timeline's times are the cut's times.
46
+ */
47
+ const floorFrame = (t) => Math.floor(t * FPS + 1e-6) / FPS;
48
+ const ceilFrame = (t) => Math.ceil(t * FPS - 1e-6) / FPS;
33
49
  const stereo = "aresample=48000,aformat=channel_layouts=stereo";
34
- /** Per-segment loudness, the level every clip was normalized to before the join. */
35
- const LOUDNORM = "loudnorm=I=-16:TP=-1.5:LRA=11,aresample=48000";
50
+ /**
51
+ * Per-segment loudness, the level every clip was normalized to before the join.
52
+ * loudnorm stamps its output about 0.1 s late (its look-ahead), so a later
53
+ * atrim by time kept that much less sound than picture and every join pulled
54
+ * the sound further ahead (#2387). Re-stamping by sample count undoes it.
55
+ * Its input is padded to 3 s, its look-ahead window: Debian's ffmpeg 5.1 (the
56
+ * worker's) all but silenced a shorter sound (#3006). Every use trims after.
57
+ */
58
+ const LOUDNORM = "apad=whole_dur=3,loudnorm=I=-16:TP=-1.5:LRA=11,aresample=48000,asetpts=N/SR/TB";
59
+ /** A fade this short at each edge of a segment's sound stops a hard cut in the room tone clicking. */
60
+ const EDGE_FADE_SEC = 0.015;
36
61
  /** Air after the last spoken word of a narrated scene, so the line lands. */
37
62
  export const TAIL_AIR_SEC = 0.8;
38
63
  /** A continuous low bed under the whole ad, so the joins between clips do not read as dead air. */
39
64
  const ROOM_TONE =
40
65
  "anoisesrc=color=brown:amplitude=0.0035:r=48000,highpass=f=60,lowpass=f=900,aformat=channel_layouts=stereo";
41
66
  const MUSIC_VOLUME = 0.18;
67
+ /** The longest a reaction beat holds the spine (#2357); older pulls carry 4 s reaction clips. */
68
+ export const REACTION_MAX_SEC = 1.5;
69
+ /**
70
+ * A reaction's laugh carries into the next line instead of stopping dead at
71
+ * the join (#2406): it starts well under the speaker, at this share of its own
72
+ * level (about -9 dB), and fades to silence over this long.
73
+ */
74
+ export const REACTION_TAIL_SEC = 0.75;
75
+ export const REACTION_TAIL_GAIN = 0.35;
76
+ /** A reaction clip with too little sound for a tail fades out over this long instead (#2666). */
77
+ export const REACTION_FADE_OUT_SEC = 0.4;
78
+ /** The least picture a reaction beat keeps when it gives up its end to a tail (#2700). */
79
+ export const REACTION_MIN_BEAT_SEC = 1.0;
80
+ /**
81
+ * The air a join keeps around its words (#2388): after one clip's last word, and
82
+ * before the next clip's first, less when the same speaker carries on than when
83
+ * someone new answers.
84
+ */
85
+ export const JOIN_TAIL_SEC = 0.15;
86
+ export const JOIN_LEAD_SAME_SPEAKER_SEC = 0.1;
87
+ export const JOIN_LEAD_NEW_SPEAKER_SEC = 0.25;
88
+ /** The gap a cut keeps from an extra heard sound it leaves out (#2541). */
89
+ const EXTRA_CLEARANCE_SEC = 0.05;
42
90
 
43
91
  /** The word rule the planner sized every line with (scout models.ts wordCount). */
44
92
  export function wordCount(text) {
@@ -54,6 +102,242 @@ export function lineWordCounts(storyboard) {
54
102
  return counts;
55
103
  }
56
104
 
105
+ const wordToken = (word) => String(word ?? "").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "");
106
+
107
+ /**
108
+ * #2541: the indexes of the heard words that deliver the first and last
109
+ * scripted word, matching the script in order at its earliest delivery, or null
110
+ * when the whole script is not heard that way.
111
+ */
112
+ export function scriptedSpan(words, lineTexts) {
113
+ const script = lineTexts.flatMap((text) => (text ?? "").split(/\s+/)).map(wordToken).filter(Boolean);
114
+ const at = heardAt(words, script);
115
+ return at && at.length > 0 ? { first: at[0], last: at.at(-1) } : null;
116
+ }
117
+
118
+ /** The heard word that delivers each script token, in order at its earliest delivery, or null. */
119
+ function heardAt(words, tokens) {
120
+ const at = [];
121
+ let j = 0;
122
+ for (const token of tokens) {
123
+ while (j < words.length && wordToken(words[j].w) !== token) j++;
124
+ if (j === words.length) return null;
125
+ at.push(j++);
126
+ }
127
+ return at;
128
+ }
129
+
130
+ /** #2544: a silence between two sentences longer than this is cut down to the kept length. */
131
+ export const SENTENCE_GAP_MAX_SEC = 0.9;
132
+ export const SENTENCE_GAP_KEEP_SEC = 0.8;
133
+ /** A pause the script writes in: an ellipsis, a dash, or a hyphen standing between words. */
134
+ const WRITTEN_PAUSE = /…|\.{3}|[—–-]/;
135
+ const SENTENCE_END = /[.?!]/;
136
+ /**
137
+ * #2554: loudness windows, and the levels that read as a sound and as quiet;
138
+ * SETTLED_WINDOWS quiet windows in a row (200 ms) end a sound (scout trim.ts).
139
+ */
140
+ const LEVEL_WINDOW_SEC = 0.02;
141
+ const SOUND_DB = -35;
142
+ const QUIET_DB = -50;
143
+ const SETTLED_WINDOWS = 10;
144
+ /**
145
+ * #2666: in a gap gone quiet, a fainter sound counts when it stands this far
146
+ * over the gap's median level and reaches FAINT_SOUND_DB, so a grunt in near
147
+ * silence goes while a soft breath and room tone stay (scout trim.ts).
148
+ */
149
+ const SOUND_OVER_FLOOR_DB = 25;
150
+ const FAINT_SOUND_DB = -45;
151
+ /** The directions that ask for a pause (scout clip-qc.ts PAUSE_DIRECTION, NO_PAUSE_DIRECTION, lineWrittenPause). */
152
+ const PAUSE_DIRECTION =
153
+ /\bpaus(e|es|ed|ing)\b|\bsilence\b|\(beat\)|\b(a|one|brief|short|quick|small|little|long)\s+beat\b|\bbeat\s*(,|before|after|then)/i;
154
+ const NO_PAUSE_DIRECTION =
155
+ /\b(no|not|without|never|don'?t|avoid\w*)\b[^.;]*\b(beats?|paus\w*|silence)\b|\bpause-?(free|less)\b/i;
156
+
157
+ export function asksPause(line) {
158
+ const direction = `${line?.stageDirection ?? ""} ${line?.performanceDirection ?? ""}`;
159
+ return PAUSE_DIRECTION.test(direction) && !NO_PAUSE_DIRECTION.test(direction);
160
+ }
161
+
162
+ /**
163
+ * #2544: the silences inside one clip to cut, as [{ s, e, after, gap }] in the
164
+ * clip's own time. A heard silence over SENTENCE_GAP_MAX_SEC between two
165
+ * scripted words at a sentence break (a new line, or . ? ! between them) keeps
166
+ * half of SENTENCE_GAP_KEEP_SEC after the earlier word and half before the
167
+ * later one; the rest goes. Kept: a pause the script writes in (a mark between
168
+ * the words, or a direction on either line asking for one), the laugh beat
169
+ * after a line a reaction is cued to, and any gap with another heard sound in
170
+ * it. Nothing is cut when the script is not heard in order.
171
+ *
172
+ * #2554: given the clip's loudness ({ windowSec, db } from 0), a sound in a
173
+ * break that holds no heard word is cut too, whatever the break's length, and
174
+ * the silence left follows the same rule. Every span of one break carries that
175
+ * break's gap, the silence it keeps, and whether a sound went.
176
+ *
177
+ * lines: [{ text, asksPause, cuesBeat }] in the order the clip speaks them.
178
+ */
179
+ export function sentenceGapCuts(words, lines, levels = null) {
180
+ const tokens = [];
181
+ let between = "";
182
+ lines.forEach((line, li) => {
183
+ for (const piece of (line.text ?? "").split(/\s+/)) {
184
+ const token = wordToken(piece);
185
+ if (!token) {
186
+ between += ` ${piece}`;
187
+ continue;
188
+ }
189
+ const [, lead, trail] = piece.match(/^([^\p{L}\p{N}]*).*?([^\p{L}\p{N}]*)$/u);
190
+ tokens.push({ token, li, before: between + lead });
191
+ between = trail;
192
+ }
193
+ between += " ";
194
+ });
195
+ const at = heardAt(words ?? [], tokens.map((t) => t.token));
196
+ if (!at) return [];
197
+ const cuts = [];
198
+ for (let t = 0; t + 1 < tokens.length; t++) {
199
+ const [a, b] = [tokens[t], tokens[t + 1]];
200
+ if (at[t + 1] !== at[t] + 1) continue;
201
+ const newLine = a.li !== b.li;
202
+ if (!newLine && !SENTENCE_END.test(b.before)) continue;
203
+ if (WRITTEN_PAUSE.test(b.before) || lines[a.li].asksPause || lines[b.li].asksPause) continue;
204
+ if (newLine && lines[a.li].cuesBeat) continue;
205
+ const [earlier, later] = [words[at[t]], words[at[t + 1]]];
206
+ const gap = later.s - earlier.e;
207
+ const heard = levels && soundBetween(levels, earlier.e, later.s);
208
+ // A sound's frame edges widen the cut, so none of it is left in.
209
+ const sound = heard && { s: Math.max(floorFrame(heard.s), ceilFrame(earlier.e)), e: ceilFrame(heard.e) };
210
+ const removed = sound ? sound.e - sound.s : 0;
211
+ const spans = sound ? [sound] : [];
212
+ if (gap - removed > SENTENCE_GAP_MAX_SEC + 1e-6) {
213
+ // Placed on the gap with the sound already out, then moved back past it.
214
+ // Frame edges only ever widen what is kept, so the silence left stays
215
+ // between the kept length and the longest allowed.
216
+ const back = (t) => (sound && t > sound.s ? t + removed : t);
217
+ const s = back(ceilFrame(earlier.e + SENTENCE_GAP_KEEP_SEC / 2));
218
+ const e = back(floorFrame(later.s - removed - SENTENCE_GAP_KEEP_SEC / 2));
219
+ if (e > s) spans.push({ s, e });
220
+ }
221
+ const merged = [];
222
+ for (const span of spans.sort((x, y) => x.s - y.s)) {
223
+ const prev = merged.at(-1);
224
+ if (prev && span.s <= prev.e + 1e-6) prev.e = Math.max(prev.e, span.e);
225
+ else merged.push({ ...span });
226
+ }
227
+ const kept = gap - merged.reduce((sum, c) => sum + c.e - c.s, 0);
228
+ for (const { s, e } of merged) cuts.push({ s, e, after: earlier.w, gap, kept, sound: Boolean(sound) });
229
+ }
230
+ return cuts;
231
+ }
232
+
233
+ /**
234
+ * #2554: a sound between two words that holds no heard word, from where the
235
+ * earlier word fades under QUIET_DB to where SETTLED_WINDOWS of quiet resume
236
+ * before the later word, or null when there is none or it runs into the later
237
+ * word. A sound heard only against the gap's quiet (#2666) must come after the
238
+ * gap settles, so the earlier word's own release stays. Mirrors scout trim.ts
239
+ * soundBetween.
240
+ */
241
+ function soundBetween(levels, fromSec, toSec) {
242
+ const { db, windowSec } = levels;
243
+ const windowAt = (sec) => Math.floor(sec / windowSec + 1e-9);
244
+ const last = Math.min(windowAt(toSec), db.length);
245
+ let quiet = windowAt(fromSec);
246
+ while (quiet < last && db[quiet] >= QUIET_DB) quiet++;
247
+ const gap = db.slice(quiet, last).sort((a, b) => a - b);
248
+ const faint = Math.min(SOUND_DB, Math.max(FAINT_SOUND_DB, gap[gap.length >> 1] + SOUND_OVER_FLOOR_DB));
249
+ const calm = settledQuiet(db, quiet);
250
+ const loud = (i) => db[i] >= (calm >= 0 && i >= calm + SETTLED_WINDOWS ? faint : SOUND_DB);
251
+ let resumed = -1;
252
+ for (let i = quiet; i < last; i = resumed + SETTLED_WINDOWS) {
253
+ while (i < last && !loud(i)) i++;
254
+ if (i >= last) break;
255
+ const settled = settledQuiet(db, i);
256
+ if (settled < 0 || settled + SETTLED_WINDOWS > last) break;
257
+ resumed = settled;
258
+ }
259
+ return resumed < 0 ? null : { s: quiet * windowSec, e: resumed * windowSec };
260
+ }
261
+
262
+ /** Scanning forward from `from`, the first window of the first run of SETTLED_WINDOWS quiet windows, or -1. */
263
+ function settledQuiet(db, from) {
264
+ let run = 0;
265
+ for (let i = from; i < db.length; i++) {
266
+ run = db[i] < QUIET_DB ? run + 1 : 0;
267
+ if (run === SETTLED_WINDOWS) return i - (SETTLED_WINDOWS - 1);
268
+ }
269
+ return -1;
270
+ }
271
+
272
+ /** Per-20 ms RMS loudness (dBFS) of mono s16 PCM, digital silence reading -120 (scout ffmpeg-clip.ts audioLevelsFromPcm). */
273
+ export function levelsFromPcm(pcm, sampleRate) {
274
+ const perWindow = Math.round(sampleRate * LEVEL_WINDOW_SEC);
275
+ const db = [];
276
+ for (let from = 0; from < pcm.length; from += perWindow) {
277
+ const to = Math.min(pcm.length, from + perWindow);
278
+ let sumSq = 0;
279
+ for (let i = from; i < to; i++) sumSq += pcm[i] * pcm[i];
280
+ const rms = Math.sqrt(sumSq / (to - from)) / 32768;
281
+ db.push(rms > 0 ? Math.max(-120, 20 * Math.log10(rms)) : -120);
282
+ }
283
+ return { windowSec: LEVEL_WINDOW_SEC, db };
284
+ }
285
+
286
+ /** A clip's loudness windows, or null when its audio cannot be decoded; `filter` shapes the sound first. */
287
+ function clipLevels(file, filter = null) {
288
+ const res = spawnSync(
289
+ "ffmpeg",
290
+ [
291
+ "-v", "error", "-i", file, "-map", "0:a:0", "-ac", "1", "-ar", "16000",
292
+ ...(filter ? ["-af", filter] : []), "-f", "s16le", "-",
293
+ ],
294
+ { maxBuffer: 1 << 28 },
295
+ );
296
+ if (res.status !== 0 || !res.stdout?.length) return null;
297
+ const bytes = new Uint8Array(res.stdout.subarray(0, res.stdout.length & ~1));
298
+ return levelsFromPcm(new Int16Array(bytes.buffer), 16000);
299
+ }
300
+
301
+ /**
302
+ * Take each own-sound clip's long sentence gaps out of its timeline: its words
303
+ * move onto the cut timeline and it runs shorter, so every later step (join
304
+ * trims, reaction splits, cue times, captions) works in cut time. The renderer
305
+ * removes seg.cuts from the picture and the sound before anything else.
306
+ */
307
+ function tightenSentenceGaps(spine, reactionCues, scriptLines, readLevels) {
308
+ for (const seg of spine) {
309
+ if (seg.source !== "clip" || seg.audio !== "clip" || !seg.words?.length) continue;
310
+ const lines = seg.lineIds.map((id) => ({
311
+ text: scriptLines.get(id)?.text,
312
+ asksPause: asksPause(scriptLines.get(id)),
313
+ cuesBeat: reactionCues.has(id),
314
+ }));
315
+ const cuts = sentenceGapCuts(seg.words, lines, readLevels(seg.file));
316
+ if (cuts.length === 0) continue;
317
+ const removedBefore = (t) => cuts.reduce((sum, c) => sum + (c.e <= t ? c.e - c.s : 0), 0);
318
+ const removed = removedBefore(Infinity);
319
+ seg.cuts = cuts.map(({ s, e }) => ({ s, e }));
320
+ seg.words = seg.words.map((w) => ({ ...w, s: w.s - removedBefore(w.s), e: w.e - removedBefore(w.s) }));
321
+ seg.seconds -= removed;
322
+ seg.clipSeconds -= removed;
323
+ seg.note = withHow(
324
+ seg.note,
325
+ [
326
+ ...new Set(
327
+ cuts.map((c) =>
328
+ c.sound
329
+ ? `sound after "${c.after}" cut, pause ${c.gap.toFixed(2)}s -> ${c.kept.toFixed(2)}s`
330
+ : `pause after "${c.after}" tightened ${c.gap.toFixed(2)}s -> ${c.kept.toFixed(2)}s`,
331
+ ),
332
+ ),
333
+ ],
334
+ );
335
+ }
336
+ }
337
+
338
+ const withHow = (note, how) =>
339
+ note.endsWith(")") ? `${note.slice(0, -1)}, ${how.join(", ")})` : `${note} (${how.join(", ")})`;
340
+
57
341
  const CUTAWAY_KINDS = new Set(["cutaway", "insert"]);
58
342
 
59
343
  const hasText = (value) => typeof value === "string" && value.trim().length > 0;
@@ -77,12 +361,19 @@ export function scenePlans(storyboard) {
77
361
  const plans = new Map();
78
362
  for (const scene of storyboard?.scenes ?? []) {
79
363
  if (typeof scene?.sceneIndex !== "number") continue;
364
+ const speakers = (Array.isArray(scene.dialogue) ? scene.dialogue : [])
365
+ .filter((turn) => hasText(turn?.line) || hasText(turn?.text))
366
+ .map((turn) => (hasText(turn.speaker) ? turn.speaker.trim() : null));
80
367
  plans.set(scene.sceneIndex, {
368
+ firstSpeaker: speakers[0] ?? null,
369
+ lastSpeaker: speakers.at(-1) ?? null,
81
370
  cutaway: CUTAWAY_KINDS.has(scene.kind),
82
371
  narrated: narratedScene(scene, storyboard),
83
372
  lineIds: Array.isArray(scene.lineIds) ? scene.lineIds : [],
84
373
  cueLineId: typeof scene.cueLineId === "string" ? scene.cueLineId : null,
374
+ cutawayType: typeof scene.cutawayType === "string" ? scene.cutawayType : null,
85
375
  durationSec: typeof scene.durationSec === "number" ? scene.durationSec : null,
376
+ actionTailSec: typeof scene.actionTailSec === "number" && scene.actionTailSec > 0 ? scene.actionTailSec : 0,
86
377
  });
87
378
  }
88
379
  return plans;
@@ -90,13 +381,16 @@ export function scenePlans(storyboard) {
90
381
 
91
382
  /**
92
383
  * Fit a picture to its voice track. The segment always runs as long as the
93
- * voice plus a beat of air, because the voice is never altered. A clip that
94
- * runs longer is trimmed; a clip that runs shorter holds its last frame for the
95
- * difference. A still (no clip) simply runs the voice's length.
384
+ * voice plus a beat of air, to the next whole frame, because the voice is never
385
+ * altered. A clip that runs longer is trimmed; a clip that runs shorter holds
386
+ * its last frame for the difference. A still (no clip) simply runs the voice's length.
387
+ * A scene marked with actionTailSec keeps that long after the voice, up to the clip's
388
+ * end, so the set-down it ends on is seen (#3199).
96
389
  */
97
- export function fitNarration({ clipSeconds, voiceSeconds }) {
98
- const target = voiceSeconds + TAIL_AIR_SEC;
99
- if (clipSeconds === null) return { seconds: target, holdSec: 0, trimmedBy: 0 };
390
+ export function fitNarration({ clipSeconds, voiceSeconds, actionTailSec = 0 }) {
391
+ const aired = ceilFrame(voiceSeconds + TAIL_AIR_SEC);
392
+ if (clipSeconds === null) return { seconds: aired, holdSec: 0, trimmedBy: 0 };
393
+ const target = Math.max(aired, Math.min(clipSeconds, ceilFrame(voiceSeconds + actionTailSec)));
100
394
  if (target <= clipSeconds) {
101
395
  return { seconds: target, holdSec: 0, trimmedBy: clipSeconds - target };
102
396
  }
@@ -126,18 +420,176 @@ export function cueOffsetOnSpine(spine, counts, cueLineId) {
126
420
  return null;
127
421
  }
128
422
 
129
- export function buildTimeline({ manifest, storyboard, skip, probe, exists, readWords }) {
423
+ /**
424
+ * Where a segment's spoken cue line ends, in seconds into the segment, or null
425
+ * when the segment does not speak it. A cue line that closes the segment ends
426
+ * where the segment does.
427
+ */
428
+ export function cueEndInSegment(seg, counts, cueLineId) {
429
+ const idx = seg.lineIds.indexOf(cueLineId);
430
+ if (idx === -1) return null;
431
+ if (idx === seg.lineIds.length - 1) return seg.seconds;
432
+ let through = 0;
433
+ let totalWords = 0;
434
+ seg.lineIds.forEach((id, i) => {
435
+ const n = counts.get(id) ?? 0;
436
+ totalWords += n;
437
+ if (i <= idx) through += n;
438
+ });
439
+ if (seg.words && seg.words.length > 0) {
440
+ const last = seg.words[Math.min(Math.max(through - 1, 0), seg.words.length - 1)];
441
+ return Math.min(Math.max(0, last?.e ?? 0), seg.seconds);
442
+ }
443
+ return totalWords > 0 ? (through / totalWords) * seg.seconds : seg.seconds;
444
+ }
445
+
446
+ /**
447
+ * Cut the dead air out of each join between clips that play their own sound: a
448
+ * clip's lead-in silence stacked on the previous clip's tail ran past a second
449
+ * (#2388). Each side comes back to a beat that reads as one conversation, a
450
+ * longer one when someone new answers. Only ever shortens, never into a word.
451
+ * The ad's first lead-in comes back to the same-speaker beat too, since a clip
452
+ * can hum before its first word (#2666); its last tail is left alone, and a
453
+ * clip whose closing line cues a reaction is not the ad's end, since the beat
454
+ * follows it. Trims land on whole frames so the picture and its sound stay the
455
+ * same length.
456
+ *
457
+ * The beat is measured from the clip's scripted words when the script is heard
458
+ * in order, and a cut stops short of a sound made up before or after the line
459
+ * ("Hmm", "Um"), even at the ad's own edges (#2541). Sounds inside the line stay.
460
+ * A scene the storyboard marks with actionTailSec keeps that long after its last
461
+ * word, so the action it ends on (her setting the mat down, #3199) is seen.
462
+ */
463
+ function trimJoins(spine, reactionCues, scriptLines) {
464
+ spine.forEach((seg, k) => {
465
+ if (seg.source !== "clip" || seg.audio !== "clip" || !seg.words?.length) return;
466
+ const prev = spine[k - 1];
467
+ const sameSpeaker = Boolean(prev?.lastSpeaker) && prev.lastSpeaker === seg.firstSpeaker;
468
+ const lead = sameSpeaker || !prev ? JOIN_LEAD_SAME_SPEAKER_SEC : JOIN_LEAD_NEW_SPEAKER_SEC;
469
+ const span = scriptedSpan(seg.words, seg.lineIds.map((id) => scriptLines.get(id)?.text));
470
+ const line = span ? seg.words.slice(span.first, span.last + 1) : seg.words;
471
+ const before = span ? seg.words.slice(0, span.first) : [];
472
+ const after = span ? seg.words.slice(span.last + 1) : [];
473
+ const firstWord = Math.min(...line.map((w) => w.s));
474
+ const lastWord = Math.max(...line.map((w) => w.e));
475
+ let inSec = Math.max(0, floorFrame(firstWord - lead));
476
+ if (before.length > 0) {
477
+ const clear = ceilFrame(Math.max(...before.map((w) => w.e)) + EXTRA_CLEARANCE_SEC);
478
+ inSec = Math.max(inSec, Math.min(clear, floorFrame(firstWord)));
479
+ }
480
+ const endsAd = k === spine.length - 1 && !reactionCues.has(seg.lineIds.at(-1));
481
+ let outSec = endsAd
482
+ ? after.length > 0 ? ceilFrame(lastWord + TAIL_AIR_SEC) : seg.seconds
483
+ : ceilFrame(lastWord + Math.max(JOIN_TAIL_SEC, seg.actionTailSec ?? 0));
484
+ if (after.length > 0) {
485
+ const clear = floorFrame(Math.min(...after.map((w) => w.s)) - EXTRA_CLEARANCE_SEC);
486
+ outSec = Math.min(outSec, Math.max(clear, ceilFrame(lastWord)));
487
+ }
488
+ outSec = Math.min(seg.seconds, outSec);
489
+ if (outSec <= inSec) return;
490
+ const how = [];
491
+ if (inSec > 0) how.push(`lead-in trimmed ${inSec.toFixed(2)}s`);
492
+ if (outSec < seg.seconds) how.push(`tail trimmed ${(seg.seconds - outSec).toFixed(2)}s`);
493
+ if (how.length === 0) return;
494
+ const kept = seg.words.filter((w) => w.s < outSec && w.e > inSec);
495
+ if (before.some((w) => !kept.includes(w))) how.push("extra sound before the line trimmed");
496
+ if (after.some((w) => !kept.includes(w))) how.push("extra sound after the line trimmed");
497
+ seg.inSec = inSec;
498
+ seg.seconds = outSec - inSec;
499
+ seg.words = kept.map((w) => ({ ...w, s: w.s - inSec, e: w.e - inSec }));
500
+ seg.note = withHow(seg.note, how);
501
+ });
502
+ }
503
+
504
+ const holdFor = (seg) =>
505
+ seg.source === "clip" ? Math.max(0, seg.seconds - Math.max(0, seg.clipSeconds - seg.inSec)) : 0;
506
+
507
+ /**
508
+ * Put a reaction beat into the spine right after its cue line ends, splitting
509
+ * the segment that speaks it when more lines follow in that segment. Returns
510
+ * the scene that speaks the cue, or null when no segment does.
511
+ */
512
+ function placeReaction(spine, counts, reaction) {
513
+ for (let k = 0; k < spine.length; k++) {
514
+ const seg = spine[k];
515
+ if (seg.reaction) continue;
516
+ const cueEnd = cueEndInSegment(seg, counts, reaction.cueLineId);
517
+ if (cueEnd === null) continue;
518
+ const cut = Math.min(seg.seconds, ceilFrame(cueEnd));
519
+ const beat = {
520
+ sceneIndex: reaction.sceneIndex, source: reaction.source, file: reaction.file,
521
+ clipSeconds: reaction.source === "clip" ? reaction.seconds : null, inSec: 0,
522
+ seconds: Math.min(reaction.seconds, REACTION_MAX_SEC), holdSec: 0,
523
+ audio: reaction.hasAudio ? "clip" : "silence", voice: null, words: null, lineIds: [],
524
+ reaction: { cueLineId: reaction.cueLineId, cueSceneIndex: seg.sceneIndex },
525
+ };
526
+ if (cut >= seg.seconds - 0.05) {
527
+ spine.splice(k + 1, 0, beat);
528
+ return seg.sceneIndex;
529
+ }
530
+ const idx = seg.lineIds.indexOf(reaction.cueLineId);
531
+ const head = {
532
+ ...seg, seconds: cut, lineIds: seg.lineIds.slice(0, idx + 1),
533
+ words: seg.words ? seg.words.filter((w) => w.s < cut) : null,
534
+ note: `${seg.note}, until ${reaction.cueLineId} ends at ${cut.toFixed(2)}s`,
535
+ };
536
+ const tail = {
537
+ ...seg, inSec: seg.inSec + cut, seconds: seg.seconds - cut, lineIds: seg.lineIds.slice(idx + 1),
538
+ words: seg.words
539
+ ? seg.words.filter((w) => w.s >= cut).map((w) => ({ ...w, s: w.s - cut, e: w.e - cut }))
540
+ : null,
541
+ note: `${seg.note}, resumed at ${cut.toFixed(2)}s after reaction ${reaction.sceneIndex}`,
542
+ };
543
+ head.holdSec = holdFor(head);
544
+ tail.holdSec = holdFor(tail);
545
+ spine.splice(k, 1, head, beat, tail);
546
+ return seg.sceneIndex;
547
+ }
548
+ return null;
549
+ }
550
+
551
+ /**
552
+ * The sound a reaction beat carries under the next segment (#2406), as a J/L
553
+ * cut: the picture moves on early and the clip's sound from where it stops
554
+ * plays under the next line. A clip that ends with its beat gives up the
555
+ * beat's end, down to REACTION_MIN_BEAT_SEC, rather than replaying sound the
556
+ * beat already played, which was heard as a stutter (#2666, #2700). Shortens
557
+ * the beat, so it runs before the spine's start times are set. Null when
558
+ * under 0.4 s of sound would carry over; the beat fades itself out then.
559
+ */
560
+ function carveReactionTail(seg, next) {
561
+ const clipLen = seg.clipSeconds ?? seg.seconds;
562
+ const overlap =
563
+ Math.round(Math.min(REACTION_TAIL_SEC, clipLen - seg.inSec - REACTION_MIN_BEAT_SEC, next.seconds) * 1000) / 1000;
564
+ if (overlap < REACTION_FADE_OUT_SEC) return null;
565
+ seg.seconds = Math.min(seg.seconds, Math.round((clipLen - seg.inSec - overlap) * 1000) / 1000);
566
+ return { fromSec: seg.inSec + seg.seconds, seconds: overlap, gain: REACTION_TAIL_GAIN };
567
+ }
568
+
569
+ /**
570
+ * #2666: how long a reaction beat fades in. A generated laugh clip can hum
571
+ * before the laugh starts, so the fade runs across that quiet start, up to its
572
+ * first sound; without the clip's loudness it is the plain edge fade.
573
+ */
574
+ function quietStart(levels, seconds) {
575
+ const first = levels ? levels.db.findIndex((d) => d >= SOUND_DB) : -1;
576
+ return first > 0 ? Math.min(seconds, Math.max(EDGE_FADE_SEC, first * levels.windowSec)) : EDGE_FADE_SEC;
577
+ }
578
+
579
+ export function buildTimeline({ manifest, storyboard, skip, probe, exists, readWords, readLevels = () => null }) {
130
580
  const plans = scenePlans(storyboard);
131
581
  const counts = lineWordCounts(storyboard);
132
582
  const spine = [];
133
583
  const cutaways = [];
134
584
  const waiting = [];
135
585
  const dropped = [];
136
- let startSec = 0;
137
586
 
138
587
  for (const scene of manifest?.scenes ?? []) {
139
588
  const n = scene.sceneIndex;
140
- const plan = plans.get(n) ?? { cutaway: false, lineIds: [], cueLineId: null, durationSec: null };
589
+ const plan = plans.get(n) ?? {
590
+ cutaway: false, lineIds: [], cueLineId: null, cutawayType: null, durationSec: null,
591
+ firstSpeaker: null, lastSpeaker: null,
592
+ };
141
593
  if (skip.has(n)) {
142
594
  dropped.push({ n, reason: `scene ${n}: skipped by --skip` });
143
595
  continue;
@@ -145,15 +597,17 @@ export function buildTimeline({ manifest, storyboard, skip, probe, exists, readW
145
597
  const nothing = `nothing to cut with (clip ${scene.clipStatus}${scene.error ? `: ${scene.error}` : ""})`;
146
598
 
147
599
  if (plan.cutaway) {
600
+ const cue = { cueLineId: plan.cueLineId, reaction: plan.cutawayType === "reaction" };
148
601
  if (exists(scene.clip)) {
602
+ const clip = probe(scene.clip);
149
603
  waiting.push({
150
604
  sceneIndex: n, source: "clip", file: scene.clip,
151
- seconds: probe(scene.clip).seconds, cueLineId: plan.cueLineId,
605
+ seconds: clip.seconds, hasAudio: clip.hasAudio, ...cue,
152
606
  });
153
607
  } else if (exists(scene.keyframe)) {
154
608
  waiting.push({
155
609
  sceneIndex: n, source: "still", file: scene.keyframe,
156
- seconds: plan.durationSec ?? 4, cueLineId: plan.cueLineId,
610
+ seconds: plan.durationSec ?? 4, hasAudio: false, ...cue,
157
611
  });
158
612
  } else {
159
613
  dropped.push({ n, reason: `cutaway ${n}: ${nothing}` });
@@ -168,11 +622,14 @@ export function buildTimeline({ manifest, storyboard, skip, probe, exists, readW
168
622
  if (exists(scene.clip)) {
169
623
  const clip = probe(scene.clip);
170
624
  const base = {
171
- sceneIndex: n, source: "clip", file: scene.clip, clipSeconds: clip.seconds,
172
- startSec, lineIds: plan.lineIds,
625
+ sceneIndex: n, source: "clip", file: scene.clip, clipSeconds: clip.seconds, inSec: 0,
626
+ lineIds: plan.lineIds, firstSpeaker: plan.firstSpeaker, lastSpeaker: plan.lastSpeaker,
627
+ actionTailSec: plan.actionTailSec ?? 0,
173
628
  };
174
629
  if (voice && (narrated || !clip.hasAudio)) {
175
- const fit = fitNarration({ clipSeconds: clip.seconds, voiceSeconds: probe(voice).seconds });
630
+ const fit = fitNarration({
631
+ clipSeconds: clip.seconds, voiceSeconds: probe(voice).seconds, actionTailSec: plan.actionTailSec ?? 0,
632
+ });
176
633
  const how = [];
177
634
  if (fit.trimmedBy > 0.05) how.push(`clip trimmed ${clip.seconds.toFixed(1)}s -> ${fit.seconds.toFixed(1)}s`);
178
635
  if (fit.holdSec > 0.05) how.push(`last frame held ${fit.holdSec.toFixed(1)}s`);
@@ -198,7 +655,6 @@ export function buildTimeline({ manifest, storyboard, skip, probe, exists, readW
198
655
  words: null, note: `${flag}, silent clip`,
199
656
  });
200
657
  }
201
- startSec += spine[spine.length - 1].seconds;
202
658
  continue;
203
659
  }
204
660
  if (exists(scene.keyframe)) {
@@ -206,39 +662,72 @@ export function buildTimeline({ manifest, storyboard, skip, probe, exists, readW
206
662
  ? fitNarration({ clipSeconds: null, voiceSeconds: probe(voice).seconds })
207
663
  : { seconds: plan.durationSec ?? 4, holdSec: 0 };
208
664
  spine.push({
209
- sceneIndex: n, source: "still", file: scene.keyframe, clipSeconds: null,
665
+ sceneIndex: n, source: "still", file: scene.keyframe, clipSeconds: null, inSec: 0,
210
666
  seconds: fit.seconds, audio: voice ? "narration" : "silence", voice, holdSec: 0,
211
- startSec, lineIds: plan.lineIds,
667
+ lineIds: plan.lineIds,
212
668
  // #1689: on a narrated scene the words are the voice track's, and the
213
669
  // voice starts where the segment does, so they need no offset here.
214
670
  words: voice ? readWords(scene.words) : null,
215
671
  note: `${flag}${voice ? ` + ${voice}` : ", silent"} (${fit.seconds.toFixed(1)}s, clip ${scene.clipStatus})`,
216
672
  });
217
- startSec += fit.seconds;
218
673
  continue;
219
674
  }
220
675
  dropped.push({ n, reason: `scene ${n}: ${nothing}` });
221
676
  }
222
677
 
223
- for (const c of waiting) {
678
+ const noCue = (c) =>
679
+ c.cueLineId ? `${c.cueLineId} is spoken by no scene in the cut` : "the storyboard names no cue line";
680
+
681
+ // Sentence gaps and then joins are closed first so a reaction beat lands
682
+ // right after its cue line ends in the trimmed clip. Reactions change the
683
+ // spine, so they go in before any overlay is placed on it.
684
+ const scriptLines = new Map((storyboard?.script ?? []).map((line) => [line?.lineId, line]));
685
+ const reactionCues = new Set(waiting.filter((w) => w.reaction).map((w) => w.cueLineId));
686
+ tightenSentenceGaps(spine, reactionCues, scriptLines, readLevels);
687
+ trimJoins(spine, reactionCues, scriptLines);
688
+ for (const c of waiting.filter((w) => w.reaction)) {
689
+ if (!c.cueLineId || placeReaction(spine, counts, c) === null) {
690
+ dropped.push({ n: c.sceneIndex, reason: `cutaway ${c.sceneIndex}: ${noCue(c)}` });
691
+ }
692
+ }
693
+ spine.forEach((seg, k) => {
694
+ if (!seg.reaction || seg.audio !== "clip") return;
695
+ const tail = spine[k + 1] ? carveReactionTail(seg, spine[k + 1]) : null;
696
+ if (tail) seg.reaction.tail = tail;
697
+ else seg.fadeOutSec = Math.min(REACTION_FADE_OUT_SEC, seg.seconds);
698
+ seg.fadeInSec = quietStart(readLevels(seg.file), seg.seconds);
699
+ });
700
+ let total = 0;
701
+ for (const seg of spine) {
702
+ seg.startSec = total;
703
+ total += seg.seconds;
704
+ }
705
+ spine.forEach((seg, k) => {
706
+ if (seg.reaction?.tail) seg.reaction.tail.atSec = spine[k + 1].startSec;
707
+ });
708
+
709
+ for (const c of waiting.filter((w) => !w.reaction)) {
224
710
  const cue = c.cueLineId ? cueOffsetOnSpine(spine, counts, c.cueLineId) : null;
225
711
  if (!cue) {
226
- const why = c.cueLineId
227
- ? `${c.cueLineId} is spoken by no scene in the cut`
228
- : "the storyboard names no cue line";
229
- dropped.push({ n: c.sceneIndex, reason: `cutaway ${c.sceneIndex}: ${why}` });
712
+ dropped.push({ n: c.sceneIndex, reason: `cutaway ${c.sceneIndex}: ${noCue(c)}` });
230
713
  continue;
231
714
  }
232
715
  cutaways.push({ ...c, startSec: cue.startSec, method: cue.method, spineSceneIndex: cue.spineSceneIndex });
233
716
  }
234
717
  cutaways.sort((a, b) => a.startSec - b.startSec);
718
+ const beats = spine.filter((seg) => seg.reaction);
235
719
  for (let i = 0; i < cutaways.length; i++) {
236
720
  const c = cutaways[i];
721
+ const stops = [];
237
722
  const next = cutaways[i + 1];
238
- const end = Math.min(startSec, next ? next.startSec : Infinity);
239
- if (c.startSec + c.seconds > end) {
240
- c.seconds = Math.max(0, end - c.startSec);
241
- c.cutShortBy = next && end === next.startSec ? `cutaway ${next.sceneIndex} starts` : "the ad ends";
723
+ if (next) stops.push({ at: next.startSec, by: `cutaway ${next.sceneIndex} starts` });
724
+ const beat = beats.find((b) => b.startSec >= c.startSec);
725
+ if (beat) stops.push({ at: beat.startSec, by: `reaction ${beat.sceneIndex} starts` });
726
+ stops.push({ at: total, by: "the ad ends" });
727
+ const stop = stops.reduce((a, b) => (b.at < a.at ? b : a));
728
+ if (c.startSec + c.seconds > stop.at) {
729
+ c.seconds = Math.max(0, stop.at - c.startSec);
730
+ c.cutShortBy = stop.by;
242
731
  }
243
732
  }
244
733
 
@@ -262,6 +751,7 @@ export function buildFfmpegArgs(timeline, { resolve, music, out }) {
262
751
  };
263
752
  const filters = [];
264
753
  const pairs = [];
754
+ const tails = [];
265
755
 
266
756
  for (const seg of timeline.spine) {
267
757
  const i =
@@ -269,26 +759,76 @@ export function buildFfmpegArgs(timeline, { resolve, music, out }) {
269
759
  ? addInput("-i", resolve(seg.file))
270
760
  : addInput("-loop", "1", "-t", String(seg.seconds), "-i", resolve(seg.file));
271
761
  const fit = [];
272
- if (seg.source === "clip" && seg.clipSeconds !== null && seg.seconds < seg.clipSeconds - 0.01) {
273
- fit.push(`trim=0:${sec(seg.seconds)},setpts=PTS-STARTPTS`);
762
+ const inSec = seg.inSec ?? 0;
763
+ // #2544: sentence gaps come out first, the picture by frame number and the
764
+ // sound by the same times, so everything after works on the cut timeline.
765
+ const cuts = seg.cuts ?? [];
766
+ let heard = `[${i}:a]${stereo},`;
767
+ if (cuts.length > 0) {
768
+ const frames = cuts.map((c) => `between(n\\,${Math.round(c.s * FPS)}\\,${Math.round(c.e * FPS) - 1})`);
769
+ fit.push(`select=not(${frames.join("+")}),setpts=N/FRAME_RATE/TB`);
770
+ const kept = [0, ...cuts.flatMap((c) => [c.s, c.e])];
771
+ const pieces = cuts.length + 1;
772
+ filters.push(`[${i}:a]${stereo},asplit=${pieces}${Array.from({ length: pieces }, (_, p) => `[ap${i}_${p}]`).join("")}`);
773
+ for (let p = 0; p < pieces; p++) {
774
+ const end = p < cuts.length ? `:end=${kept[2 * p + 1].toFixed(6)}` : "";
775
+ filters.push(`[ap${i}_${p}]atrim=start=${kept[2 * p].toFixed(6)}${end},asetpts=PTS-STARTPTS[aq${i}_${p}]`);
776
+ }
777
+ filters.push(
778
+ `${Array.from({ length: pieces }, (_, p) => `[aq${i}_${p}]`).join("")}concat=n=${pieces}:v=0:a=1[ac${i}]`,
779
+ );
780
+ heard = `[ac${i}]`;
781
+ }
782
+ if (seg.source === "clip" && seg.clipSeconds !== null) {
783
+ const shown = seg.seconds - seg.holdSec;
784
+ if (inSec > 0.001 || shown < seg.clipSeconds - 0.01) {
785
+ // By frame number: a time printed to the millisecond can round past the
786
+ // frame it names and drop it.
787
+ const from = Math.round(inSec * FPS);
788
+ fit.push(`trim=start_frame=${from}:end_frame=${Math.round((inSec + shown) * FPS)},setpts=PTS-STARTPTS`);
789
+ }
274
790
  }
275
791
  if (seg.holdSec > 0.01) fit.push(`tpad=stop_mode=clone:stop_duration=${sec(seg.holdSec)}`);
276
792
  filters.push(`${norm(`${i}:v`)}${fit.length ? `,${fit.join(",")}` : ""}[v${i}]`);
793
+ // A segment split around a reaction beat starts partway into its sound too.
794
+ const skipIn = inSec > 0.001 ? `atrim=start=${sec(inSec)},asetpts=PTS-STARTPTS,` : "";
795
+ const [fadeIn, fadeOut] = [seg.fadeInSec ?? EDGE_FADE_SEC, seg.fadeOutSec ?? EDGE_FADE_SEC];
796
+ // A cubic rise keeps a reaction's quiet start near silent until its laugh (#2666).
797
+ const rise = fadeIn > EDGE_FADE_SEC ? ":curve=cub" : "";
798
+ const fitLength =
799
+ `apad,atrim=0:${sec(seg.seconds)},afade=t=in:d=${sec(fadeIn)}${rise},` +
800
+ `afade=t=out:st=${sec(seg.seconds - fadeOut)}:d=${sec(fadeOut)}`;
801
+ const fitSound = `${LOUDNORM},${fitLength}`;
277
802
  if (seg.audio === "narration") {
278
803
  const v = addInput("-i", resolve(seg.voice));
279
- filters.push(`[${v}:a]${stereo},${LOUDNORM},apad,atrim=0:${sec(seg.seconds)}[a${i}]`);
804
+ filters.push(`[${v}:a]${stereo},${skipIn}${fitSound}[a${i}]`);
805
+ } else if (seg.audio === "clip" && seg.reaction?.tail) {
806
+ // The tail is cut from the same loudness-normalized sound as the beat,
807
+ // ducked, faded out, and laid in at the join; the next line's own sound
808
+ // plays at full level over it (#2406).
809
+ const t = seg.reaction.tail;
810
+ filters.push(`${heard}${skipIn}${LOUDNORM},asplit=2[as${i}][at${i}]`);
811
+ filters.push(`[as${i}]${fitLength}[a${i}]`);
812
+ filters.push(
813
+ `[at${i}]atrim=start=${sec(t.fromSec - inSec)}:duration=${sec(t.seconds)},asetpts=PTS-STARTPTS,` +
814
+ `volume=${t.gain},afade=t=in:d=${EDGE_FADE_SEC},afade=t=out:st=0:d=${sec(t.seconds)},` +
815
+ `adelay=delays=${Math.round(t.atSec * 1000)}:all=1[tail${i}]`,
816
+ );
817
+ tails.push(`[tail${i}]`);
280
818
  } else if (seg.audio === "clip") {
281
- filters.push(`[${i}:a]${stereo},${LOUDNORM},apad,atrim=0:${sec(seg.seconds)}[a${i}]`);
819
+ filters.push(`${heard}${skipIn}${fitSound}[a${i}]`);
282
820
  } else {
283
821
  filters.push(`anullsrc=r=48000:cl=stereo,atrim=0:${sec(seg.seconds)}[a${i}]`);
284
822
  }
285
823
  pairs.push(`[v${i}][a${i}]`);
286
824
  }
287
825
 
288
- // A clip whose audio ends before its video would otherwise end the whole cut
289
- // there under -shortest; padding lets the video decide the length.
290
826
  filters.push(`${pairs.join("")}concat=n=${pairs.length}:v=1:a=1[vcat][voicecat]`);
291
- filters.push(`[voicecat]apad[voice]`);
827
+ // The sound is padded or cut to the picture's exact length, so every stream
828
+ // ends on its own. No -shortest: ffmpeg 8 held the whole ad's frames for it,
829
+ // and Debian's ffmpeg 5.1 (the worker's) never ended at all (#2988).
830
+ const pictureSec = timeline.spine.reduce((sum, seg) => sum + seg.seconds, 0);
831
+ filters.push(`[voicecat]apad,atrim=end=${pictureSec.toFixed(6)}[voice]`);
292
832
 
293
833
  let videoLabel = "vcat";
294
834
  timeline.cutaways.forEach((c, j) => {
@@ -308,7 +848,7 @@ export function buildFfmpegArgs(timeline, { resolve, music, out }) {
308
848
  });
309
849
 
310
850
  filters.push(`${ROOM_TONE}[room]`);
311
- const beds = ["[room]"];
851
+ const beds = ["[room]", ...tails];
312
852
  if (music) {
313
853
  const m = addInput("-stream_loop", "-1", "-i", music);
314
854
  filters.push(`[${m}:a]${stereo},volume=${MUSIC_VOLUME}[bed]`);
@@ -326,7 +866,7 @@ export function buildFfmpegArgs(timeline, { resolve, music, out }) {
326
866
  "-map", `[${videoLabel}]`, "-map", `[${audioLabel}]`,
327
867
  "-c:v", "libx264", "-preset", "medium", "-crf", "20", "-r", String(FPS),
328
868
  "-c:a", "aac", "-b:a", "192k",
329
- "-movflags", "+faststart", "-shortest",
869
+ "-movflags", "+faststart",
330
870
  out,
331
871
  ];
332
872
  }
@@ -398,6 +938,23 @@ export function inTheCut(timeline) {
398
938
  const rows = [];
399
939
  for (const seg of timeline.spine) {
400
940
  const label = seg.source === "clip" ? seg.file : `still ${seg.file}`;
941
+ if (seg.reaction) {
942
+ const sound = seg.audio === "clip" ? "its own sound" : "silent: the clip has no sound";
943
+ const tail = seg.reaction.tail;
944
+ const fadeIn =
945
+ seg.fadeInSec > EDGE_FADE_SEC ? `, fading in over its quiet first ${seg.fadeInSec.toFixed(2)}s` : "";
946
+ const fadeOut = tail
947
+ ? `, fading out under the next line at ${tail.atSec.toFixed(2)}s over ${tail.seconds.toFixed(2)}s`
948
+ : seg.fadeOutSec ? `, fading out over its last ${seg.fadeOutSec.toFixed(2)}s` : "";
949
+ const fades = fadeIn + fadeOut;
950
+ rows.push({
951
+ n: seg.sceneIndex,
952
+ line:
953
+ `reaction ${seg.sceneIndex}: ${label} for ${seg.seconds.toFixed(1)}s after ` +
954
+ `${seg.reaction.cueLineId} in scene ${seg.reaction.cueSceneIndex}, ${sound}${fades}`,
955
+ });
956
+ continue;
957
+ }
401
958
  rows.push({ n: seg.sceneIndex, line: `scene ${seg.sceneIndex}: ${label}${seg.note}` });
402
959
  }
403
960
  for (const c of timeline.cutaways) {
@@ -417,10 +974,112 @@ export function inTheCut(timeline) {
417
974
  return rows.sort((a, b) => a.n - b.n).map((r) => r.line);
418
975
  }
419
976
 
977
+ /**
978
+ * #2666: a generated clip can hum or click where nobody speaks, and the cut
979
+ * plays that stretch with nothing over it. A hum is HUM_BAND's share of the
980
+ * sound (the 1 kHz whine heard before a first word) holding for HUM_MIN_SEC
981
+ * over HUM_FLOOR_DB; a click is a sound of at most CLICK_MAX_WINDOWS with
982
+ * quiet on both sides, loud as the gap cutter reads a sound in near silence.
983
+ */
984
+ export const HUM_BAND = "highpass=f=800,highpass=f=800,lowpass=f=1500,lowpass=f=1500";
985
+ // White noise measures about -13.5 dB through HUM_BAND at 16 kHz (-10.6 dB on
986
+ // paper), and the four hums #2666 was calibrated on held -6 to -10 dB.
987
+ const HUM_SHARE_DB = -10;
988
+ const HUM_FLOOR_DB = -70;
989
+ const HUM_MIN_SEC = 0.16;
990
+ const CLICK_MAX_WINDOWS = 3;
991
+
992
+ /**
993
+ * Where the cut plays a clip's sound with no word in it, in the clip's own
994
+ * time: before a dialogue clip's first word, after its last, and a reaction's
995
+ * quiet start (under its cubic fade-in when the cut fades it).
996
+ */
997
+ function edgeStretches(seg, levels) {
998
+ const inSec = seg.inSec ?? 0;
999
+ if (seg.reaction) {
1000
+ const quiet = quietStart(levels, seg.seconds);
1001
+ if (quiet <= EDGE_FADE_SEC) return [];
1002
+ const fadeIn = seg.fadeInSec ?? 0;
1003
+ return [{ side: "before its laugh", from: inSec, to: inSec + quiet, at: seg.startSec, fadeIn }];
1004
+ }
1005
+ if (!seg.words?.length) return [];
1006
+ // inSec and the words are on the clip's timeline with its sentence gaps
1007
+ // taken out; the levels are the file's own, so each edge goes back through
1008
+ // the cuts before it.
1009
+ const raw = (t) => {
1010
+ let at = t;
1011
+ for (const c of seg.cuts ?? []) if (c.s <= at) at += c.e - c.s;
1012
+ return at;
1013
+ };
1014
+ const first = seg.words[0].s;
1015
+ const last = Math.max(...seg.words.map((w) => w.e));
1016
+ const shown = seg.seconds - (seg.holdSec ?? 0);
1017
+ return [
1018
+ { side: "before its first word", from: raw(inSec), to: raw(inSec + first), at: seg.startSec, fadeIn: 0 },
1019
+ {
1020
+ side: "after its last word", from: raw(inSec + last), to: raw(inSec + shown),
1021
+ at: seg.startSec + last, fadeIn: 0,
1022
+ },
1023
+ ];
1024
+ }
1025
+
1026
+ /**
1027
+ * The hums and clicks the cut plays at the edges of its clips, one line each,
1028
+ * for the caller to listen to. It never changes the cut. readEdgeLevels(file)
1029
+ * gives { windowSec, db, bandDb } from time 0, or null.
1030
+ */
1031
+ export function edgeSounds(timeline, readEdgeLevels) {
1032
+ const found = [];
1033
+ for (const seg of timeline.spine) {
1034
+ if (seg.source !== "clip" || seg.audio !== "clip") continue;
1035
+ const levels = readEdgeLevels(seg.file);
1036
+ if (!levels) continue;
1037
+ const { windowSec, db, bandDb } = levels;
1038
+ for (const stretch of edgeStretches(seg, levels)) {
1039
+ const from = Math.ceil(stretch.from / windowSec - 1e-9);
1040
+ const to = Math.min(db.length, Math.floor(stretch.to / windowSec + 1e-9));
1041
+ if (to - from < 2) continue;
1042
+ const heard = [];
1043
+ for (let i = from; i < to; i++) {
1044
+ const into = (i + 0.5) * windowSec - stretch.from;
1045
+ const faded = stretch.fadeIn > 0 && into < stretch.fadeIn;
1046
+ const gain = faded ? 60 * Math.log10(Math.max(1e-3, into / stretch.fadeIn)) : 0;
1047
+ heard.push({ i, db: db[i] + gain, share: bandDb[i] - db[i] });
1048
+ }
1049
+ const cutAt = (i) => stretch.at + i * windowSec - stretch.from;
1050
+ const where = `scene ${seg.sceneIndex}, ${stretch.side}`;
1051
+ let run = 0;
1052
+ let longest = { length: 0, end: 0 };
1053
+ heard.forEach((w, k) => {
1054
+ run = w.db >= HUM_FLOOR_DB && w.share >= HUM_SHARE_DB ? run + 1 : 0;
1055
+ if (run > longest.length) longest = { length: run, end: k };
1056
+ });
1057
+ if (longest.length * windowSec >= HUM_MIN_SEC - 1e-9) {
1058
+ const start = heard[longest.end - longest.length + 1].i;
1059
+ found.push(
1060
+ `${where}: a hum for ${(longest.length * windowSec).toFixed(2)}s at ${cutAt(start).toFixed(2)}s in the cut`,
1061
+ );
1062
+ }
1063
+ const sorted = heard.map((w) => w.db).sort((a, b) => a - b);
1064
+ const loud = Math.max(FAINT_SOUND_DB, sorted[sorted.length >> 1] + SOUND_OVER_FLOOR_DB);
1065
+ for (let k = 1; k < heard.length - 1; k++) {
1066
+ if (heard[k].db < loud || heard[k - 1].db >= QUIET_DB) continue;
1067
+ let end = k;
1068
+ while (end < heard.length && heard[end].db >= loud) end++;
1069
+ if (end < heard.length && heard[end].db < QUIET_DB && end - k <= CLICK_MAX_WINDOWS) {
1070
+ found.push(`${where}: a click at ${cutAt(heard[k].i).toFixed(2)}s in the cut`);
1071
+ }
1072
+ k = end;
1073
+ }
1074
+ }
1075
+ }
1076
+ return found;
1077
+ }
1078
+
420
1079
  export function noWordTimes(timeline) {
421
1080
  const rows = timeline.spine
422
1081
  .filter((seg) => !seg.words || seg.words.length === 0)
423
- .map((seg) => `scene ${seg.sceneIndex} (no words.json)`);
1082
+ .map((seg) => `scene ${seg.sceneIndex} (${seg.reaction ? "reaction" : "no words.json"})`);
424
1083
  for (const c of timeline.cutaways) rows.push(`scene ${c.sceneIndex} (cutaway)`);
425
1084
  return rows;
426
1085
  }
@@ -430,6 +1089,15 @@ function fail(code, message) {
430
1089
  process.exit(code);
431
1090
  }
432
1091
 
1092
+ /** A refusal the CLI prints to stderr and exits with `code`, so another caller can do the same or not. */
1093
+ export class FirstCutError extends Error {
1094
+ constructor(code, message) {
1095
+ super(message);
1096
+ this.name = "FirstCutError";
1097
+ this.code = code;
1098
+ }
1099
+ }
1100
+
433
1101
  function parseArgs(argv) {
434
1102
  const opts = { dir: undefined, out: undefined, skip: new Set(), music: true };
435
1103
  for (let i = 0; i < argv.length; i++) {
@@ -445,26 +1113,29 @@ function parseArgs(argv) {
445
1113
  return opts;
446
1114
  }
447
1115
 
448
- function main() {
449
- const opts = parseArgs(process.argv.slice(2));
450
-
1116
+ /**
1117
+ * The whole cut, as the CLI makes it, for any caller (#2988: the worker cuts a
1118
+ * Video-page run with this). Renders `out` (default <dir>/cut.mp4) and its word
1119
+ * sidecars, and throws FirstCutError where the CLI would exit. The CLI's report
1120
+ * is printed by main() from what this returns.
1121
+ */
1122
+ export function firstCut({ dir, out: outOpt, skip = new Set(), music: musicOn = true }) {
451
1123
  for (const bin of ["ffmpeg", "ffprobe"]) {
452
1124
  if (spawnSync(bin, ["-version"]).status !== 0) {
453
- fail(2, `${bin} is not installed. Install ffmpeg (brew install ffmpeg / apt install ffmpeg).`);
1125
+ throw new FirstCutError(2, `${bin} is not installed. Install ffmpeg (brew install ffmpeg / apt install ffmpeg).`);
454
1126
  }
455
1127
  }
456
1128
  const encoders = spawnSync("ffmpeg", ["-hide_banner", "-encoders"]).stdout?.toString() ?? "";
457
1129
  if (!/\blibx264\b/.test(encoders)) {
458
- fail(2, "This ffmpeg has no libx264 encoder (H.264). Install a full ffmpeg build: brew install ffmpeg / apt install ffmpeg.");
1130
+ throw new FirstCutError(2, "This ffmpeg has no libx264 encoder (H.264). Install a full ffmpeg build: brew install ffmpeg / apt install ffmpeg.");
459
1131
  }
460
1132
 
461
- const dir = opts.dir;
462
1133
  const manifestPath = path.join(dir, "manifest.json");
463
1134
  if (!existsSync(manifestPath)) {
464
- fail(2, `${manifestPath} not found. Run: exodus video pull <runId> --out ${dir}`);
1135
+ throw new FirstCutError(2, `${manifestPath} not found. Run: exodus video pull <runId> --out ${dir}`);
465
1136
  }
466
1137
  const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
467
- const out = path.resolve(opts.out ?? path.join(dir, "cut.mp4"));
1138
+ const out = path.resolve(outOpt ?? path.join(dir, "cut.mp4"));
468
1139
  const here = (file) => path.join(dir, file);
469
1140
  const exists = (file) => Boolean(file) && existsSync(here(file));
470
1141
 
@@ -478,11 +1149,14 @@ function main() {
478
1149
  };
479
1150
  const probe = (file) => {
480
1151
  const raw = execFileSync("ffprobe", [
481
- "-v", "error", "-show_entries", "stream=codec_type:format=duration", "-of", "json", here(file),
1152
+ "-v", "error", "-show_entries", "stream=codec_type,duration:stream_disposition=attached_pic:format=duration", "-of", "json", here(file),
482
1153
  ]).toString();
483
1154
  const info = JSON.parse(raw);
1155
+ // A clip's container runs a few ms past its picture on the audio's padding;
1156
+ // the picture is what the cut shows, so its length is the clip's length.
1157
+ const picture = Number((info.streams ?? []).find((s) => s.codec_type === "video" && !s.disposition?.attached_pic)?.duration);
484
1158
  return {
485
- seconds: Number(info.format?.duration ?? 0),
1159
+ seconds: picture > 0 ? picture : Number(info.format?.duration ?? 0),
486
1160
  hasAudio: (info.streams ?? []).some((s) => s.codec_type === "audio"),
487
1161
  };
488
1162
  };
@@ -494,19 +1168,20 @@ function main() {
494
1168
  const timeline = buildTimeline({
495
1169
  manifest,
496
1170
  storyboard,
497
- skip: opts.skip,
1171
+ skip,
498
1172
  probe,
499
1173
  exists,
500
1174
  readWords: (file) => {
501
1175
  const rows = readJson(file);
502
1176
  return Array.isArray(rows) && rows.length > 0 ? rows : null;
503
1177
  },
1178
+ readLevels: (file) => clipLevels(here(file)),
504
1179
  });
505
1180
  if (timeline.spine.length === 0) {
506
- fail(1, `No scene has a clip or a keyframe. Check: exodus video status ${manifest.runId}`);
1181
+ throw new FirstCutError(1, `No scene has a clip or a keyframe. Check: exodus video status ${manifest.runId}`);
507
1182
  }
508
1183
 
509
- const music = opts.music && exists(manifest.music) ? manifest.music : null;
1184
+ const music = musicOn && exists(manifest.music) ? manifest.music : null;
510
1185
  const ffmpegArgs = buildFfmpegArgs(timeline, {
511
1186
  resolve: here,
512
1187
  music: music ? here(music) : null,
@@ -515,7 +1190,7 @@ function main() {
515
1190
  const res = spawnSync("ffmpeg", ffmpegArgs, { stdio: "inherit" });
516
1191
  if (res.status !== 0) {
517
1192
  const quoted = ffmpegArgs.map((a) => (/[\s\[\];]/.test(a) ? `'${a}'` : a)).join(" ");
518
- fail(1, `ffmpeg failed. The command it ran:\n ffmpeg ${quoted}`);
1193
+ throw new FirstCutError(1, `ffmpeg failed. The command it ran:\n ffmpeg ${quoted}`);
519
1194
  }
520
1195
 
521
1196
  const rows = wordRows(timeline);
@@ -531,7 +1206,22 @@ function main() {
531
1206
  const duration = execFileSync("ffprobe", [
532
1207
  "-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", out,
533
1208
  ]).toString().trim();
534
- console.log(`Wrote ${out} (${Number(duration).toFixed(1)}s, ${WIDTH}x${HEIGHT})`);
1209
+ return { out, durationSec: Number(duration), manifest, timeline, music, rows, sidecars };
1210
+ }
1211
+
1212
+ function main() {
1213
+ const opts = parseArgs(process.argv.slice(2));
1214
+ let cut;
1215
+ try {
1216
+ cut = firstCut(opts);
1217
+ } catch (e) {
1218
+ if (e instanceof FirstCutError) fail(e.code, e.message);
1219
+ throw e;
1220
+ }
1221
+ const { out, durationSec, manifest, timeline, music, rows, sidecars } = cut;
1222
+ const here = (file) => path.join(opts.dir, file);
1223
+
1224
+ console.log(`Wrote ${out} (${durationSec.toFixed(1)}s, ${WIDTH}x${HEIGHT})`);
535
1225
  console.log("In the cut:");
536
1226
  for (const line of inTheCut(timeline)) console.log(" " + line);
537
1227
  if (timeline.left.length > 0) {
@@ -548,8 +1238,25 @@ function main() {
548
1238
  }
549
1239
  const silent = noWordTimes(timeline);
550
1240
  if (silent.length > 0) console.log(`No word times: ${silent.join(", ")}`);
1241
+ const edges = edgeSounds(timeline, (file) => {
1242
+ const levels = clipLevels(here(file));
1243
+ const band = levels && clipLevels(here(file), HUM_BAND);
1244
+ return band ? { ...levels, bandDb: band.db } : null;
1245
+ });
1246
+ if (edges.length > 0) {
1247
+ console.log("Sound at the edges (listen before uploading; the cut is unchanged):");
1248
+ for (const line of edges) console.log(" " + line);
1249
+ }
551
1250
  console.log(`Room tone: under the whole ad. Music bed: ${music ?? "none"}`);
552
1251
  console.log(`Upload it: exodus video upload ${manifest.runId} --file ${out}`);
553
1252
  }
554
1253
 
555
- if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) main();
1254
+ // #2988: the basename test too, because a bundler that inlines this file into
1255
+ // its entry would otherwise run the CLI inside whatever imported it.
1256
+ if (
1257
+ process.argv[1] &&
1258
+ path.basename(process.argv[1]) === "first-cut.mjs" &&
1259
+ import.meta.url === pathToFileURL(process.argv[1]).href
1260
+ ) {
1261
+ main();
1262
+ }