@aicopycoders/exodus 2026.9.2000-beta.9 → 2026.10.1200-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/docs/CLAUDE.md +1 -0
- package/assets/skills/exodus-primer/SKILL.md +10 -5
- package/assets/skills/exodus-video/SKILL.md +203 -47
- package/assets/skills/exodus-video/scripts/first-cut.d.mts +44 -0
- package/assets/skills/exodus-video/scripts/first-cut.mjs +1140 -92
- package/assets/skills/exodus-winners/SKILL.md +13 -1
- package/assets/skills/swipe/SKILL.md +78 -0
- package/dist/commands/ads.js +1 -1
- package/dist/commands/doctor.js +1 -1
- package/dist/commands/primer.js +84 -12
- package/dist/commands/video-customer.d.ts +101 -0
- package/dist/commands/video-customer.js +158 -0
- package/dist/commands/video.d.ts +169 -9
- package/dist/commands/video.js +877 -87
- package/dist/commands/whoami.js +2 -1
- package/dist/commands/workflow.d.ts +2 -1
- package/dist/commands/workflow.js +16 -2
- package/dist/lib/meta-ads.d.ts +1 -0
- package/package.json +3 -3
|
@@ -2,43 +2,114 @@
|
|
|
2
2
|
// A plain first cut from a folder written by `exodus video pull`: the A-roll
|
|
3
3
|
// scenes in the manifest's order make the spine, each cutaway lays over the
|
|
4
4
|
// spine at the moment its cued line is spoken, a room-tone bed and the music
|
|
5
|
-
// bed go underneath, and one MP4 comes out ready to upload.
|
|
5
|
+
// bed go underneath, and one MP4 comes out ready to upload. A reaction
|
|
6
|
+
// cutaway is the exception: it is its own short beat in the spine right after
|
|
7
|
+
// its cued line ends, heard with its own sound while the speaker is silent,
|
|
8
|
+
// fading in across any quiet start and out rather than stopping dead.
|
|
6
9
|
//
|
|
7
|
-
// node first-cut.mjs <pulled-dir> [--out cut.mp4] [--skip 2,5] [--no-music]
|
|
10
|
+
// node first-cut.mjs <pulled-dir> [--out cut.mp4] [--skip 2,5] [--no-music] [--still-fill]
|
|
8
11
|
//
|
|
9
12
|
// The audio track comes first. A narrated scene (the storyboard gives it
|
|
10
13
|
// narration and no on-camera line) plays its voice track and NOTHING the clip
|
|
11
14
|
// recorded: the voice replaces the clip's audio, and the picture is fitted to
|
|
12
15
|
// the voice, trimmed when the clip runs longer and held on its last frame when
|
|
13
16
|
// the voice runs longer. The voice itself is never altered. A dialogue scene
|
|
14
|
-
// keeps the audio it performed
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
//
|
|
17
|
+
// keeps the audio it performed; at each join its silence before the first word
|
|
18
|
+
// and after the last is cut back to a short beat, read from its word times.
|
|
19
|
+
// Inside it, a silence over 0.6 s between two sentences is cut down to 0.5 s,
|
|
20
|
+
// picture and sound together, unless the script writes that pause in. A
|
|
21
|
+
// sound in such a break that holds no heard word (a burp, a click) is cut
|
|
22
|
+
// out, read from the clip's loudness, and the same rule then applies. Every
|
|
23
|
+
// segment is loudness-normalized to -16 LUFS before the join.
|
|
24
|
+
// A scene with no clip stops the cut (#3365: a customer's ad once ended on 16 s
|
|
25
|
+
// of frozen, silent keyframes). --skip drops scenes. --still-fill instead turns
|
|
26
|
+
// a scene with no clip but a keyframe into a still for the length of its voice
|
|
27
|
+
// track, else the storyboard's planned duration. A run whose Pacing is jump cuts (manifest.shots)
|
|
28
|
+
// punches in on every other cut between one speaker's scenes; a montage does
|
|
29
|
+
// the same where one speaker's scenes in a row play one base shot.
|
|
18
30
|
//
|
|
19
31
|
// After the render it writes the whole ad's word times beside the MP4, as
|
|
20
|
-
// <out>.words.json and <out>.srt, for captioning in post
|
|
32
|
+
// <out>.words.json and <out>.srt, for captioning in post, and where each scene
|
|
33
|
+
// plays in the cut as <out>.scenes.json (#3300).
|
|
21
34
|
import { execFileSync, spawnSync } from "node:child_process";
|
|
22
35
|
import { existsSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
23
36
|
import path from "node:path";
|
|
24
37
|
import { pathToFileURL } from "node:url";
|
|
25
38
|
|
|
26
|
-
const USAGE = "usage: first-cut.mjs <pulled-dir> [--out cut.mp4] [--skip 2,5] [--no-music]";
|
|
39
|
+
const USAGE = "usage: first-cut.mjs <pulled-dir> [--out cut.mp4] [--skip 2,5] [--no-music] [--still-fill]";
|
|
27
40
|
const WIDTH = 1080;
|
|
28
41
|
const HEIGHT = 1920;
|
|
29
42
|
const FPS = 24;
|
|
30
43
|
const norm = (label) =>
|
|
31
44
|
`[${label}]scale=${WIDTH}:${HEIGHT}:force_original_aspect_ratio=decrease,` +
|
|
32
45
|
`pad=${WIDTH}:${HEIGHT}:(ow-iw)/2:(oh-ih)/2,setsar=1,fps=${FPS},format=yuv420p`;
|
|
46
|
+
/**
|
|
47
|
+
* Edges the cut chooses (join trims, reaction splits, narration lengths) land
|
|
48
|
+
* on whole frames, so the picture runs as long as its sound there and the
|
|
49
|
+
* timeline's times are the cut's times.
|
|
50
|
+
*/
|
|
51
|
+
/**
|
|
52
|
+
* #3304: the jump cut's punch-in, a centre crop of the frame scaled up this
|
|
53
|
+
* much. The planner holds one framing across a jump-cut run's scenes, and a
|
|
54
|
+
* montage's line-cut pieces keep their parent's (#3323), so without it those
|
|
55
|
+
* cuts would be the same picture.
|
|
56
|
+
*/
|
|
57
|
+
export const PUNCH_IN_SCALE = 1.15;
|
|
58
|
+
const even = (n) => 2 * Math.round(n / 2);
|
|
59
|
+
const PUNCH_IN =
|
|
60
|
+
`scale=${even(WIDTH * PUNCH_IN_SCALE)}:${even(HEIGHT * PUNCH_IN_SCALE)},crop=${WIDTH}:${HEIGHT},setsar=1`;
|
|
61
|
+
const floorFrame = (t) => Math.floor(t * FPS + 1e-6) / FPS;
|
|
62
|
+
const ceilFrame = (t) => Math.ceil(t * FPS - 1e-6) / FPS;
|
|
33
63
|
const stereo = "aresample=48000,aformat=channel_layouts=stereo";
|
|
34
|
-
/**
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
64
|
+
/**
|
|
65
|
+
* Per-segment loudness, the level every clip was normalized to before the join.
|
|
66
|
+
* loudnorm stamps its output about 0.1 s late (its look-ahead), so a later
|
|
67
|
+
* atrim by time kept that much less sound than picture and every join pulled
|
|
68
|
+
* the sound further ahead (#2387). Re-stamping by sample count undoes it.
|
|
69
|
+
* Its input is padded to 3 s, its look-ahead window: Debian's ffmpeg 5.1 (the
|
|
70
|
+
* worker's) all but silenced a shorter sound (#3006). Every use trims after.
|
|
71
|
+
*/
|
|
72
|
+
const LOUDNORM = "apad=whole_dur=3,loudnorm=I=-16:TP=-1.5:LRA=11,aresample=48000,asetpts=N/SR/TB";
|
|
73
|
+
/** A fade this short at each edge of a segment's sound stops a hard cut in the room tone clicking. */
|
|
74
|
+
const EDGE_FADE_SEC = 0.015;
|
|
75
|
+
/**
|
|
76
|
+
* Air after a narrator's last word. Sizing by the voice file plus 0.8 s left
|
|
77
|
+
* ~1.1 s of silence at every narrator join, since the file already runs on
|
|
78
|
+
* ~0.3 s past the word (#3384).
|
|
79
|
+
*/
|
|
80
|
+
export const NARRATION_JOIN_AIR_SEC = 0.25;
|
|
81
|
+
/**
|
|
82
|
+
* Air after the last word of the line that ends the ad, spoken or narrated, so
|
|
83
|
+
* it lands. A dialogue clip's own tail ran ~1.1 s of dead air (#3423).
|
|
84
|
+
*/
|
|
85
|
+
export const END_AIR_SEC = 0.5;
|
|
38
86
|
/** A continuous low bed under the whole ad, so the joins between clips do not read as dead air. */
|
|
39
87
|
const ROOM_TONE =
|
|
40
88
|
"anoisesrc=color=brown:amplitude=0.0035:r=48000,highpass=f=60,lowpass=f=900,aformat=channel_layouts=stereo";
|
|
41
89
|
const MUSIC_VOLUME = 0.18;
|
|
90
|
+
/** The longest a reaction beat holds the spine (#2357); older pulls carry 4 s reaction clips. */
|
|
91
|
+
export const REACTION_MAX_SEC = 1.5;
|
|
92
|
+
/**
|
|
93
|
+
* A reaction's laugh carries into the next line instead of stopping dead at
|
|
94
|
+
* the join (#2406): it starts well under the speaker, at this share of its own
|
|
95
|
+
* level (about -9 dB), and fades to silence over this long.
|
|
96
|
+
*/
|
|
97
|
+
export const REACTION_TAIL_SEC = 0.75;
|
|
98
|
+
export const REACTION_TAIL_GAIN = 0.35;
|
|
99
|
+
/** A reaction clip with too little sound for a tail fades out over this long instead (#2666). */
|
|
100
|
+
export const REACTION_FADE_OUT_SEC = 0.4;
|
|
101
|
+
/** The least picture a reaction beat keeps when it gives up its end to a tail (#2700). */
|
|
102
|
+
export const REACTION_MIN_BEAT_SEC = 1.0;
|
|
103
|
+
/**
|
|
104
|
+
* The air a join keeps around its words (#2388): after one clip's last word, and
|
|
105
|
+
* before the next clip's first, less when the same speaker carries on than when
|
|
106
|
+
* someone new answers.
|
|
107
|
+
*/
|
|
108
|
+
export const JOIN_TAIL_SEC = 0.15;
|
|
109
|
+
export const JOIN_LEAD_SAME_SPEAKER_SEC = 0.1;
|
|
110
|
+
export const JOIN_LEAD_NEW_SPEAKER_SEC = 0.25;
|
|
111
|
+
/** The gap a cut keeps from an extra heard sound it leaves out (#2541). */
|
|
112
|
+
const EXTRA_CLEARANCE_SEC = 0.05;
|
|
42
113
|
|
|
43
114
|
/** The word rule the planner sized every line with (scout models.ts wordCount). */
|
|
44
115
|
export function wordCount(text) {
|
|
@@ -54,6 +125,405 @@ export function lineWordCounts(storyboard) {
|
|
|
54
125
|
return counts;
|
|
55
126
|
}
|
|
56
127
|
|
|
128
|
+
const wordToken = (word) => String(word ?? "").toLowerCase().replace(/[^\p{L}\p{N}]+/gu, "");
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* #3175: one spoken word a script and a scribe spell differently ("ok", "Okay").
|
|
132
|
+
* #3460: "alright" maps to the scribe's "All right" once its halves are joined.
|
|
133
|
+
*/
|
|
134
|
+
const SAME_WORD_SPELLINGS = new Map([
|
|
135
|
+
["ok", "okay"],
|
|
136
|
+
["alright", "allright"],
|
|
137
|
+
]);
|
|
138
|
+
|
|
139
|
+
/** The spoken tokens of one written word; empty for pure punctuation. */
|
|
140
|
+
function spokenWords(word) {
|
|
141
|
+
const core = String(word ?? "").toLowerCase().replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
|
|
142
|
+
// #3133: a scribe writes a spoken "glowmist dot com" as "Glowmist.com".
|
|
143
|
+
const parts = /^[a-z0-9]+(?:-[a-z0-9]+)*(?:\.[a-z]{2,})+$/.test(core)
|
|
144
|
+
? core.split(".").flatMap((part, i) => (i ? ["dot", part] : [part]))
|
|
145
|
+
: [core];
|
|
146
|
+
return parts.map(wordToken).filter(Boolean).map((t) => SAME_WORD_SPELLINGS.get(t) ?? t);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** { tokens, sourceIndex }: tokens[i] came from written word sourceIndex[i]. */
|
|
150
|
+
function spokenTokens(words) {
|
|
151
|
+
const tokens = [];
|
|
152
|
+
const sourceIndex = [];
|
|
153
|
+
words.forEach((word, i) => {
|
|
154
|
+
for (const token of spokenWords(word)) {
|
|
155
|
+
tokens.push(token);
|
|
156
|
+
sourceIndex.push(i);
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
return { tokens, sourceIndex };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** An ellipsis or a dash, which a writer may type with no space around it (scout scribe.ts PAUSE_MARK). */
|
|
163
|
+
const PAUSE_MARK = /…|\.{3,}|[—–]|-{2,}/g;
|
|
164
|
+
const scriptWords = (text) => (text ?? "").replace(PAUSE_MARK, " $& ").split(/\s+/).filter(Boolean);
|
|
165
|
+
|
|
166
|
+
const MAX_JOINED_WORDS = 3;
|
|
167
|
+
const MIN_SLIP_WORD_LENGTH = 7;
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* #2300: a scribe writes a made-up name the script spells as one word
|
|
171
|
+
* ("FreshKeep") as two ("Fresh keep"), or the other way round. Joins each run
|
|
172
|
+
* of 2-3 neighbouring tokens whose concatenation is a token of the other side,
|
|
173
|
+
* longest first. #2520: a run one slip off one of the other side's names joins
|
|
174
|
+
* too, unless a join also starts at the next token, or (#3058) its first token
|
|
175
|
+
* is a word the other side already has. Mirrors scout scribe.ts joinSplitWords.
|
|
176
|
+
*/
|
|
177
|
+
function joinSplitWords(side, otherSide, otherNames = []) {
|
|
178
|
+
const other = new Set(otherSide);
|
|
179
|
+
const fuzzy = (r) => otherNames.some((name) => sameWord(name, r, true));
|
|
180
|
+
const run = (i, k) => (i + k <= side.tokens.length ? side.tokens.slice(i, i + k).join("") : undefined);
|
|
181
|
+
const joinStartsAt = (i) => {
|
|
182
|
+
for (let k = MAX_JOINED_WORDS; k >= 2; k--) {
|
|
183
|
+
const r = run(i, k);
|
|
184
|
+
if (r !== undefined && (other.has(r) || fuzzy(r))) return true;
|
|
185
|
+
}
|
|
186
|
+
return false;
|
|
187
|
+
};
|
|
188
|
+
const tokens = [];
|
|
189
|
+
const sourceIndex = [];
|
|
190
|
+
const lastSourceIndex = [];
|
|
191
|
+
for (let i = 0; i < side.tokens.length; ) {
|
|
192
|
+
let take = 1;
|
|
193
|
+
for (let k = MAX_JOINED_WORDS; k >= 2; k--) {
|
|
194
|
+
const r = run(i, k);
|
|
195
|
+
if (r === undefined) continue;
|
|
196
|
+
if (other.has(r) || (fuzzy(r) && !joinStartsAt(i + 1) && !other.has(side.tokens[i]))) {
|
|
197
|
+
take = k;
|
|
198
|
+
break;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
tokens.push(side.tokens.slice(i, i + take).join(""));
|
|
202
|
+
sourceIndex.push(side.sourceIndex[i]);
|
|
203
|
+
lastSourceIndex.push(side.sourceIndex[i + take - 1]);
|
|
204
|
+
i += take;
|
|
205
|
+
}
|
|
206
|
+
return { tokens, sourceIndex, lastSourceIndex };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Which written words read as names: a capital inside the word, or a leading
|
|
211
|
+
* capital where no sentence starts. "I" and "I'm" are never names.
|
|
212
|
+
*/
|
|
213
|
+
function nameLikeWords(words) {
|
|
214
|
+
return words.map((word, i) => {
|
|
215
|
+
const bare = word.replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, "");
|
|
216
|
+
if (/^I(['’][a-z]+)?$/.test(bare)) return false;
|
|
217
|
+
if (/[A-Z]/.test(bare.slice(1))) return true;
|
|
218
|
+
const sentenceStart = i === 0 || /[.!?]["')\]]*$/.test(words[i - 1]);
|
|
219
|
+
return /^[A-Z]/.test(bare) && !sentenceStart;
|
|
220
|
+
});
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* #2441: one letter off on a long name is a slip; #3035: so is a spelling of
|
|
225
|
+
* the same sound, and #3415 a final z after a vowel for an s ("Inez", "Ines").
|
|
226
|
+
*/
|
|
227
|
+
function sameWord(scripted, heard, nameLike) {
|
|
228
|
+
if (scripted === heard || soundKey(scripted) === soundKey(heard)) return true;
|
|
229
|
+
if (!nameLike || scripted.length < MIN_SLIP_WORD_LENGTH) return false;
|
|
230
|
+
const [short, long] = scripted.length <= heard.length ? [scripted, heard] : [heard, scripted];
|
|
231
|
+
if (long.length - short.length > 1) return false;
|
|
232
|
+
let i = 0;
|
|
233
|
+
while (i < short.length && short[i] === long[i]) i++;
|
|
234
|
+
const rest = short.length === long.length ? i + 1 : i;
|
|
235
|
+
return short.slice(rest) === long.slice(i + 1);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const soundKey = (token) =>
|
|
239
|
+
token
|
|
240
|
+
.replace(/ph/g, "f")
|
|
241
|
+
.replace(/ck|q|c(?![eiy])/g, "k")
|
|
242
|
+
.replace(/(?<=\w{3})oe$/, "o")
|
|
243
|
+
.replace(/(?<=[aeiou])z$/, "s");
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Where the script's first complete, in-order delivery sits in the heard
|
|
247
|
+
* words, or null. heardTokenOf/heardWordOf give the heard token and word that
|
|
248
|
+
* deliver each (joined) script token; spokenWords lists every heard word that
|
|
249
|
+
* carries a token. #3140: one scripted word misheard as another is paired with
|
|
250
|
+
* the heard word in its place when the next scripted word follows it, but an
|
|
251
|
+
* exact delivery wins. Mirrors scout scribe.ts alignScript.
|
|
252
|
+
*/
|
|
253
|
+
function alignScript(words, written) {
|
|
254
|
+
const scripted = spokenTokens(written);
|
|
255
|
+
const heardWords = spokenTokens(words.map((w) => w.w));
|
|
256
|
+
const nameLike = nameLikeWords(written);
|
|
257
|
+
const names = scripted.tokens.filter((_, t) => nameLike[scripted.sourceIndex[t]]);
|
|
258
|
+
const heard = joinSplitWords(heardWords, scripted.tokens, names);
|
|
259
|
+
const expected = joinSplitWords(scripted, heard.tokens);
|
|
260
|
+
if (expected.tokens.length === 0 || heard.tokens.length === 0) return null;
|
|
261
|
+
const matches = (e, h) =>
|
|
262
|
+
h < heard.tokens.length && sameWord(expected.tokens[e], heard.tokens[h], nameLike[expected.sourceIndex[e]]);
|
|
263
|
+
const pair = (allowMisheard) => {
|
|
264
|
+
const tokenOf = [];
|
|
265
|
+
let j = 0;
|
|
266
|
+
for (let e = 0; e < expected.tokens.length; e++) {
|
|
267
|
+
const misheard = allowMisheard && !matches(e, j) && e + 1 < expected.tokens.length && matches(e + 1, j + 1);
|
|
268
|
+
if (!misheard) while (j < heard.tokens.length && !matches(e, j)) j++;
|
|
269
|
+
if (j === heard.tokens.length) return null;
|
|
270
|
+
tokenOf.push(j++);
|
|
271
|
+
}
|
|
272
|
+
return tokenOf;
|
|
273
|
+
};
|
|
274
|
+
const heardTokenOf = pair(false) ?? pair(true);
|
|
275
|
+
if (!heardTokenOf) return null;
|
|
276
|
+
return {
|
|
277
|
+
expected,
|
|
278
|
+
heard,
|
|
279
|
+
heardTokenOf,
|
|
280
|
+
heardWordOf: heardTokenOf.map((j) => heard.sourceIndex[j]),
|
|
281
|
+
spokenWords: [...new Set(heardWords.sourceIndex)],
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* #2541: the indexes of the heard words that deliver the first and last
|
|
287
|
+
* scripted word, matching the script in order at its earliest delivery, or null
|
|
288
|
+
* when the whole script is not heard that way. #3058: a last word heard split
|
|
289
|
+
* in two ends on its second half.
|
|
290
|
+
*/
|
|
291
|
+
export function scriptedSpan(words, lineTexts) {
|
|
292
|
+
const aligned = alignScript(words, lineTexts.flatMap(scriptWords));
|
|
293
|
+
if (!aligned) return null;
|
|
294
|
+
const { heard, heardWordOf, heardTokenOf } = aligned;
|
|
295
|
+
return { first: heardWordOf[0], last: heard.lastSourceIndex[heardTokenOf.at(-1)] };
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/** #2544, #3423: a silence between two sentences longer than this is cut down to the kept length. */
|
|
299
|
+
export const SENTENCE_GAP_MAX_SEC = 0.6;
|
|
300
|
+
export const SENTENCE_GAP_KEEP_SEC = 0.5;
|
|
301
|
+
/** A pause the script writes in: an ellipsis, a dash, or a hyphen standing between words. */
|
|
302
|
+
const WRITTEN_PAUSE = /…|\.{3}|[—–-]/;
|
|
303
|
+
const SENTENCE_END = /[.?!]/;
|
|
304
|
+
/** A heard word that ends a sentence; a heard ellipsis does not (scout scribe.ts HEARD_SENTENCE_END). */
|
|
305
|
+
const HEARD_SENTENCE_END = /(?<!\.)[.?!]["'”’)\]]*$/;
|
|
306
|
+
/** A heard abbreviation keeps its period mid-sentence ("2p.m.", "U.S.", "Dr."). */
|
|
307
|
+
const HEARD_ABBREVIATION = /^[\d:]*(?:\p{L}\.){2,}$|^(?:mr|mrs|ms|dr|st|vs|etc|jr|sr)\.$/iu;
|
|
308
|
+
|
|
309
|
+
function heardSentenceEnd(word) {
|
|
310
|
+
const bare = String(word ?? "").replace(/^["'“‘(\[]+|["'”’)\]]+$/g, "");
|
|
311
|
+
return HEARD_SENTENCE_END.test(bare) && !HEARD_ABBREVIATION.test(bare);
|
|
312
|
+
}
|
|
313
|
+
/**
|
|
314
|
+
* #2554: loudness windows, and the levels that read as a sound and as quiet;
|
|
315
|
+
* SETTLED_WINDOWS quiet windows in a row (200 ms) end a sound (scout trim.ts).
|
|
316
|
+
*/
|
|
317
|
+
const LEVEL_WINDOW_SEC = 0.02;
|
|
318
|
+
const SOUND_DB = -35;
|
|
319
|
+
const QUIET_DB = -50;
|
|
320
|
+
const SETTLED_WINDOWS = 10;
|
|
321
|
+
/**
|
|
322
|
+
* #2666: in a gap gone quiet, a fainter sound counts when it stands this far
|
|
323
|
+
* over the gap's median level and reaches FAINT_SOUND_DB, so a grunt in near
|
|
324
|
+
* silence goes while a soft breath and room tone stay (scout trim.ts).
|
|
325
|
+
*/
|
|
326
|
+
const SOUND_OVER_FLOOR_DB = 25;
|
|
327
|
+
const FAINT_SOUND_DB = -45;
|
|
328
|
+
/** The directions that ask for a pause (scout clip-qc.ts PAUSE_DIRECTION, NO_PAUSE_DIRECTION, lineWrittenPause). */
|
|
329
|
+
const PAUSE_DIRECTION =
|
|
330
|
+
/\bpaus(e|es|ed|ing)\b|\bsilence\b|\(beat\)|\b(a|one|brief|short|quick|small|little|long)\s+beat\b|\bbeat\s*(,|before|after|then)/i;
|
|
331
|
+
const NO_PAUSE_DIRECTION =
|
|
332
|
+
/\b(no|not|without|never|don'?t|avoid\w*)\b[^.;]*\b(beats?|paus\w*|silence)\b|\bpause-?(free|less)\b/i;
|
|
333
|
+
|
|
334
|
+
export function asksPause(line) {
|
|
335
|
+
const direction = `${line?.stageDirection ?? ""} ${line?.performanceDirection ?? ""}`;
|
|
336
|
+
return PAUSE_DIRECTION.test(direction) && !NO_PAUSE_DIRECTION.test(direction);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* #2544: the silences inside one clip to cut, as [{ s, e, after, gap }] in the
|
|
341
|
+
* clip's own time. A heard silence over SENTENCE_GAP_MAX_SEC between two
|
|
342
|
+
* scripted words at a sentence break (a new line, or . ? ! between them) keeps
|
|
343
|
+
* half of SENTENCE_GAP_KEEP_SEC after the earlier word and half before the
|
|
344
|
+
* later one; the rest goes. #3460: a speaker can end a sentence where the
|
|
345
|
+
* script writes a comma, so an earlier word the scribe heard end one counts.
|
|
346
|
+
* Kept: a pause the script writes in (a mark between the words, or a
|
|
347
|
+
* direction on either line asking for one), the laugh beat
|
|
348
|
+
* after a line a reaction is cued to, and any gap with another heard sound in
|
|
349
|
+
* it. Nothing is cut when the script is not heard in order.
|
|
350
|
+
*
|
|
351
|
+
* #2554: given the clip's loudness ({ windowSec, db } from 0), a sound in a
|
|
352
|
+
* break that holds no heard word is cut too, whatever the break's length, and
|
|
353
|
+
* the silence left follows the same rule. Every span of one break carries that
|
|
354
|
+
* break's gap, the silence it keeps, and whether a sound went.
|
|
355
|
+
*
|
|
356
|
+
* lines: [{ text, asksPause, cuesBeat }] in the order the clip speaks them.
|
|
357
|
+
*/
|
|
358
|
+
export function sentenceGapCuts(words, lines, levels = null) {
|
|
359
|
+
const written = [];
|
|
360
|
+
const pieces = [];
|
|
361
|
+
let between = "";
|
|
362
|
+
lines.forEach((line, li) => {
|
|
363
|
+
for (const piece of scriptWords(line.text)) {
|
|
364
|
+
written.push(piece);
|
|
365
|
+
if (!wordToken(piece)) {
|
|
366
|
+
between += ` ${piece}`;
|
|
367
|
+
pieces.push(null);
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
const [, lead, trail] = piece.match(/^([^\p{L}\p{N}]*).*?([^\p{L}\p{N}]*)$/u);
|
|
371
|
+
pieces.push({ li, before: between + lead });
|
|
372
|
+
between = trail;
|
|
373
|
+
}
|
|
374
|
+
between += " ";
|
|
375
|
+
});
|
|
376
|
+
const aligned = alignScript(words ?? [], written);
|
|
377
|
+
if (!aligned) return [];
|
|
378
|
+
const { expected, heardTokenOf, heardWordOf, spokenWords: spoken } = aligned;
|
|
379
|
+
const cuts = [];
|
|
380
|
+
for (let t = 0; t + 1 < heardWordOf.length; t++) {
|
|
381
|
+
if (heardTokenOf[t + 1] !== heardTokenOf[t] + 1) continue;
|
|
382
|
+
const to = heardWordOf[t + 1];
|
|
383
|
+
if (heardWordOf[t] === to) continue;
|
|
384
|
+
// A joined script token keeps its first word's line and lead-in.
|
|
385
|
+
const [a, b] = [pieces[expected.sourceIndex[t]], pieces[expected.sourceIndex[t + 1]]];
|
|
386
|
+
const newLine = a.li !== b.li;
|
|
387
|
+
// A name heard split in two (#2300) ends on its last heard word.
|
|
388
|
+
const [earlier, later] = [words[spoken.findLast((w) => w < to)], words[to]];
|
|
389
|
+
if (!newLine && !SENTENCE_END.test(b.before) && !heardSentenceEnd(earlier.w)) continue;
|
|
390
|
+
if (WRITTEN_PAUSE.test(b.before) || lines[a.li].asksPause || lines[b.li].asksPause) continue;
|
|
391
|
+
if (newLine && lines[a.li].cuesBeat) continue;
|
|
392
|
+
const gap = later.s - earlier.e;
|
|
393
|
+
const heard = levels && soundBetween(levels, earlier.e, later.s);
|
|
394
|
+
// A sound's frame edges widen the cut, so none of it is left in.
|
|
395
|
+
const sound = heard && { s: Math.max(floorFrame(heard.s), ceilFrame(earlier.e)), e: ceilFrame(heard.e) };
|
|
396
|
+
const removed = sound ? sound.e - sound.s : 0;
|
|
397
|
+
const spans = sound ? [sound] : [];
|
|
398
|
+
if (gap - removed > SENTENCE_GAP_MAX_SEC + 1e-6) {
|
|
399
|
+
// Placed on the gap with the sound already out, then moved back past it.
|
|
400
|
+
// Frame edges only ever widen what is kept, so the silence left stays
|
|
401
|
+
// between the kept length and the longest allowed.
|
|
402
|
+
const back = (t) => (sound && t > sound.s ? t + removed : t);
|
|
403
|
+
const s = back(ceilFrame(earlier.e + SENTENCE_GAP_KEEP_SEC / 2));
|
|
404
|
+
const e = back(floorFrame(later.s - removed - SENTENCE_GAP_KEEP_SEC / 2));
|
|
405
|
+
if (e > s) spans.push({ s, e });
|
|
406
|
+
}
|
|
407
|
+
const merged = [];
|
|
408
|
+
for (const span of spans.sort((x, y) => x.s - y.s)) {
|
|
409
|
+
const prev = merged.at(-1);
|
|
410
|
+
if (prev && span.s <= prev.e + 1e-6) prev.e = Math.max(prev.e, span.e);
|
|
411
|
+
else merged.push({ ...span });
|
|
412
|
+
}
|
|
413
|
+
const kept = gap - merged.reduce((sum, c) => sum + c.e - c.s, 0);
|
|
414
|
+
for (const { s, e } of merged) cuts.push({ s, e, after: earlier.w, gap, kept, sound: Boolean(sound) });
|
|
415
|
+
}
|
|
416
|
+
return cuts;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* #2554: a sound between two words that holds no heard word, from where the
|
|
421
|
+
* earlier word fades under QUIET_DB to where SETTLED_WINDOWS of quiet resume
|
|
422
|
+
* before the later word, or null when there is none or it runs into the later
|
|
423
|
+
* word. A sound heard only against the gap's quiet (#2666) must come after the
|
|
424
|
+
* gap settles, so the earlier word's own release stays. Mirrors scout trim.ts
|
|
425
|
+
* soundBetween.
|
|
426
|
+
*/
|
|
427
|
+
function soundBetween(levels, fromSec, toSec) {
|
|
428
|
+
const { db, windowSec } = levels;
|
|
429
|
+
const windowAt = (sec) => Math.floor(sec / windowSec + 1e-9);
|
|
430
|
+
const last = Math.min(windowAt(toSec), db.length);
|
|
431
|
+
let quiet = windowAt(fromSec);
|
|
432
|
+
while (quiet < last && db[quiet] >= QUIET_DB) quiet++;
|
|
433
|
+
const gap = db.slice(quiet, last).sort((a, b) => a - b);
|
|
434
|
+
const faint = Math.min(SOUND_DB, Math.max(FAINT_SOUND_DB, gap[gap.length >> 1] + SOUND_OVER_FLOOR_DB));
|
|
435
|
+
const calm = settledQuiet(db, quiet);
|
|
436
|
+
const loud = (i) => db[i] >= (calm >= 0 && i >= calm + SETTLED_WINDOWS ? faint : SOUND_DB);
|
|
437
|
+
let resumed = -1;
|
|
438
|
+
for (let i = quiet; i < last; i = resumed + SETTLED_WINDOWS) {
|
|
439
|
+
while (i < last && !loud(i)) i++;
|
|
440
|
+
if (i >= last) break;
|
|
441
|
+
const settled = settledQuiet(db, i);
|
|
442
|
+
if (settled < 0 || settled + SETTLED_WINDOWS > last) break;
|
|
443
|
+
resumed = settled;
|
|
444
|
+
}
|
|
445
|
+
return resumed < 0 ? null : { s: quiet * windowSec, e: resumed * windowSec };
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
/** Scanning forward from `from`, the first window of the first run of SETTLED_WINDOWS quiet windows, or -1. */
|
|
449
|
+
function settledQuiet(db, from) {
|
|
450
|
+
let run = 0;
|
|
451
|
+
for (let i = from; i < db.length; i++) {
|
|
452
|
+
run = db[i] < QUIET_DB ? run + 1 : 0;
|
|
453
|
+
if (run === SETTLED_WINDOWS) return i - (SETTLED_WINDOWS - 1);
|
|
454
|
+
}
|
|
455
|
+
return -1;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/** Per-20 ms RMS loudness (dBFS) of mono s16 PCM, digital silence reading -120 (scout ffmpeg-clip.ts audioLevelsFromPcm). */
|
|
459
|
+
export function levelsFromPcm(pcm, sampleRate) {
|
|
460
|
+
const perWindow = Math.round(sampleRate * LEVEL_WINDOW_SEC);
|
|
461
|
+
const db = [];
|
|
462
|
+
for (let from = 0; from < pcm.length; from += perWindow) {
|
|
463
|
+
const to = Math.min(pcm.length, from + perWindow);
|
|
464
|
+
let sumSq = 0;
|
|
465
|
+
for (let i = from; i < to; i++) sumSq += pcm[i] * pcm[i];
|
|
466
|
+
const rms = Math.sqrt(sumSq / (to - from)) / 32768;
|
|
467
|
+
db.push(rms > 0 ? Math.max(-120, 20 * Math.log10(rms)) : -120);
|
|
468
|
+
}
|
|
469
|
+
return { windowSec: LEVEL_WINDOW_SEC, db };
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/** A clip's loudness windows, or null when its audio cannot be decoded; `filter` shapes the sound first. */
|
|
473
|
+
function clipLevels(file, filter = null) {
|
|
474
|
+
const res = spawnSync(
|
|
475
|
+
"ffmpeg",
|
|
476
|
+
[
|
|
477
|
+
"-v", "error", "-i", file, "-map", "0:a:0", "-ac", "1", "-ar", "16000",
|
|
478
|
+
...(filter ? ["-af", filter] : []), "-f", "s16le", "-",
|
|
479
|
+
],
|
|
480
|
+
{ maxBuffer: 1 << 28 },
|
|
481
|
+
);
|
|
482
|
+
if (res.status !== 0 || !res.stdout?.length) return null;
|
|
483
|
+
const bytes = new Uint8Array(res.stdout.subarray(0, res.stdout.length & ~1));
|
|
484
|
+
return levelsFromPcm(new Int16Array(bytes.buffer), 16000);
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
/**
|
|
488
|
+
* Take each own-sound clip's long sentence gaps out of its timeline: its words
|
|
489
|
+
* move onto the cut timeline and it runs shorter, so every later step (join
|
|
490
|
+
* trims, reaction splits, cue times, captions) works in cut time. The renderer
|
|
491
|
+
* removes seg.cuts from the picture and the sound before anything else.
|
|
492
|
+
*/
|
|
493
|
+
function tightenSentenceGaps(spine, reactionCues, scriptLines, readLevels) {
|
|
494
|
+
for (const seg of spine) {
|
|
495
|
+
if (seg.source !== "clip" || seg.audio !== "clip" || !seg.words?.length) continue;
|
|
496
|
+
const lines = seg.lineIds.map((id) => ({
|
|
497
|
+
text: scriptLines.get(id)?.text,
|
|
498
|
+
asksPause: asksPause(scriptLines.get(id)),
|
|
499
|
+
cuesBeat: reactionCues.has(id),
|
|
500
|
+
}));
|
|
501
|
+
const cuts = sentenceGapCuts(seg.words, lines, readLevels(seg.file));
|
|
502
|
+
if (cuts.length === 0) continue;
|
|
503
|
+
const removedBefore = (t) => cuts.reduce((sum, c) => sum + (c.e <= t ? c.e - c.s : 0), 0);
|
|
504
|
+
const removed = removedBefore(Infinity);
|
|
505
|
+
seg.cuts = cuts.map(({ s, e }) => ({ s, e }));
|
|
506
|
+
seg.words = seg.words.map((w) => ({ ...w, s: w.s - removedBefore(w.s), e: w.e - removedBefore(w.s) }));
|
|
507
|
+
seg.seconds -= removed;
|
|
508
|
+
seg.clipSeconds -= removed;
|
|
509
|
+
seg.note = withHow(
|
|
510
|
+
seg.note,
|
|
511
|
+
[
|
|
512
|
+
...new Set(
|
|
513
|
+
cuts.map((c) =>
|
|
514
|
+
c.sound
|
|
515
|
+
? `sound after "${c.after}" cut, pause ${c.gap.toFixed(2)}s -> ${c.kept.toFixed(2)}s`
|
|
516
|
+
: `pause after "${c.after}" tightened ${c.gap.toFixed(2)}s -> ${c.kept.toFixed(2)}s`,
|
|
517
|
+
),
|
|
518
|
+
),
|
|
519
|
+
],
|
|
520
|
+
);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
const withHow = (note, how) =>
|
|
525
|
+
note.endsWith(")") ? `${note.slice(0, -1)}, ${how.join(", ")})` : `${note} (${how.join(", ")})`;
|
|
526
|
+
|
|
57
527
|
const CUTAWAY_KINDS = new Set(["cutaway", "insert"]);
|
|
58
528
|
|
|
59
529
|
const hasText = (value) => typeof value === "string" && value.trim().length > 0;
|
|
@@ -77,26 +547,38 @@ export function scenePlans(storyboard) {
|
|
|
77
547
|
const plans = new Map();
|
|
78
548
|
for (const scene of storyboard?.scenes ?? []) {
|
|
79
549
|
if (typeof scene?.sceneIndex !== "number") continue;
|
|
550
|
+
const speakers = (Array.isArray(scene.dialogue) ? scene.dialogue : [])
|
|
551
|
+
.filter((turn) => hasText(turn?.line) || hasText(turn?.text))
|
|
552
|
+
.map((turn) => (hasText(turn.speaker) ? turn.speaker.trim() : null));
|
|
80
553
|
plans.set(scene.sceneIndex, {
|
|
554
|
+
firstSpeaker: speakers[0] ?? null,
|
|
555
|
+
lastSpeaker: speakers.at(-1) ?? null,
|
|
556
|
+
baseShotId: hasText(scene.baseShotId) ? scene.baseShotId.trim() : null,
|
|
81
557
|
cutaway: CUTAWAY_KINDS.has(scene.kind),
|
|
82
558
|
narrated: narratedScene(scene, storyboard),
|
|
83
559
|
lineIds: Array.isArray(scene.lineIds) ? scene.lineIds : [],
|
|
84
560
|
cueLineId: typeof scene.cueLineId === "string" ? scene.cueLineId : null,
|
|
561
|
+
cutawayType: typeof scene.cutawayType === "string" ? scene.cutawayType : null,
|
|
85
562
|
durationSec: typeof scene.durationSec === "number" ? scene.durationSec : null,
|
|
563
|
+
actionTailSec: typeof scene.actionTailSec === "number" && scene.actionTailSec > 0 ? scene.actionTailSec : 0,
|
|
86
564
|
});
|
|
87
565
|
}
|
|
88
566
|
return plans;
|
|
89
567
|
}
|
|
90
568
|
|
|
91
569
|
/**
|
|
92
|
-
* Fit a picture to its voice track. The segment
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
570
|
+
* Fit a picture to its voice track. The segment runs to where the voice stops
|
|
571
|
+
* speaking (spokenEndSec) plus airSec, to the next whole frame; the voice is
|
|
572
|
+
* never altered, only cut in the silence after that point. A clip that runs
|
|
573
|
+
* longer is trimmed; a clip that runs shorter holds its last frame for the
|
|
574
|
+
* difference. A still (no clip) simply runs that length. A scene marked with
|
|
575
|
+
* actionTailSec keeps that long after the voice, up to the clip's end, so the
|
|
576
|
+
* set-down it ends on is seen (#3199).
|
|
96
577
|
*/
|
|
97
|
-
export function fitNarration({ clipSeconds,
|
|
98
|
-
const
|
|
99
|
-
if (clipSeconds === null) return { seconds:
|
|
578
|
+
export function fitNarration({ clipSeconds, spokenEndSec, airSec, actionTailSec = 0 }) {
|
|
579
|
+
const aired = ceilFrame(spokenEndSec + airSec);
|
|
580
|
+
if (clipSeconds === null) return { seconds: aired, holdSec: 0, trimmedBy: 0 };
|
|
581
|
+
const target = Math.max(aired, Math.min(clipSeconds, ceilFrame(spokenEndSec + actionTailSec)));
|
|
100
582
|
if (target <= clipSeconds) {
|
|
101
583
|
return { seconds: target, holdSec: 0, trimmedBy: clipSeconds - target };
|
|
102
584
|
}
|
|
@@ -126,18 +608,241 @@ export function cueOffsetOnSpine(spine, counts, cueLineId) {
|
|
|
126
608
|
return null;
|
|
127
609
|
}
|
|
128
610
|
|
|
129
|
-
|
|
611
|
+
/**
|
|
612
|
+
* Where a segment's spoken cue line ends, in seconds into the segment, or null
|
|
613
|
+
* when the segment does not speak it. A cue line that closes the segment ends
|
|
614
|
+
* where the segment does.
|
|
615
|
+
*/
|
|
616
|
+
export function cueEndInSegment(seg, counts, cueLineId) {
|
|
617
|
+
const idx = seg.lineIds.indexOf(cueLineId);
|
|
618
|
+
if (idx === -1) return null;
|
|
619
|
+
if (idx === seg.lineIds.length - 1) return seg.seconds;
|
|
620
|
+
let through = 0;
|
|
621
|
+
let totalWords = 0;
|
|
622
|
+
seg.lineIds.forEach((id, i) => {
|
|
623
|
+
const n = counts.get(id) ?? 0;
|
|
624
|
+
totalWords += n;
|
|
625
|
+
if (i <= idx) through += n;
|
|
626
|
+
});
|
|
627
|
+
if (seg.words && seg.words.length > 0) {
|
|
628
|
+
const last = seg.words[Math.min(Math.max(through - 1, 0), seg.words.length - 1)];
|
|
629
|
+
return Math.min(Math.max(0, last?.e ?? 0), seg.seconds);
|
|
630
|
+
}
|
|
631
|
+
return totalWords > 0 ? (through / totalWords) * seg.seconds : seg.seconds;
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
/**
|
|
635
|
+
* Cut the dead air out of each join between clips that play their own sound: a
|
|
636
|
+
* clip's lead-in silence stacked on the previous clip's tail ran past a second
|
|
637
|
+
* (#2388). Each side comes back to a beat that reads as one conversation, a
|
|
638
|
+
* longer one when someone new answers. Only ever shortens, never into a word.
|
|
639
|
+
* The ad's first lead-in comes back to the same-speaker beat too, since a clip
|
|
640
|
+
* can hum before its first word (#2666); its last tail comes back to
|
|
641
|
+
* END_AIR_SEC (#3423), and a clip whose closing line cues a reaction is not the
|
|
642
|
+
* ad's end, since the beat follows it. Trims land on whole frames so the
|
|
643
|
+
* picture and its sound stay the same length.
|
|
644
|
+
*
|
|
645
|
+
* The beat is measured from the clip's scripted words when the script is heard
|
|
646
|
+
* in order, and a cut stops short of a sound made up before or after the line
|
|
647
|
+
* ("Hmm", "Um"), even at the ad's own edges (#2541). Sounds inside the line stay.
|
|
648
|
+
* A scene the storyboard marks with actionTailSec keeps that long after its last
|
|
649
|
+
* word, so the action it ends on (her setting the mat down, #3199) is seen.
|
|
650
|
+
*/
|
|
651
|
+
function trimJoins(spine, reactionCues, scriptLines) {
|
|
652
|
+
spine.forEach((seg, k) => {
|
|
653
|
+
if (seg.source !== "clip" || seg.audio !== "clip" || !seg.words?.length) return;
|
|
654
|
+
const prev = spine[k - 1];
|
|
655
|
+
const sameSpeaker = Boolean(prev?.lastSpeaker) && prev.lastSpeaker === seg.firstSpeaker;
|
|
656
|
+
const lead = sameSpeaker || !prev ? JOIN_LEAD_SAME_SPEAKER_SEC : JOIN_LEAD_NEW_SPEAKER_SEC;
|
|
657
|
+
const span = scriptedSpan(seg.words, seg.lineIds.map((id) => scriptLines.get(id)?.text));
|
|
658
|
+
const line = span ? seg.words.slice(span.first, span.last + 1) : seg.words;
|
|
659
|
+
const before = span ? seg.words.slice(0, span.first) : [];
|
|
660
|
+
const after = span ? seg.words.slice(span.last + 1) : [];
|
|
661
|
+
const firstWord = Math.min(...line.map((w) => w.s));
|
|
662
|
+
const lastWord = Math.max(...line.map((w) => w.e));
|
|
663
|
+
let inSec = Math.max(0, floorFrame(firstWord - lead));
|
|
664
|
+
if (before.length > 0) {
|
|
665
|
+
const clear = ceilFrame(Math.max(...before.map((w) => w.e)) + EXTRA_CLEARANCE_SEC);
|
|
666
|
+
inSec = Math.max(inSec, Math.min(clear, floorFrame(firstWord)));
|
|
667
|
+
}
|
|
668
|
+
const endsAd = k === spine.length - 1 && !reactionCues.has(seg.lineIds.at(-1));
|
|
669
|
+
const tailSec = Math.max(endsAd ? END_AIR_SEC : JOIN_TAIL_SEC, seg.actionTailSec ?? 0);
|
|
670
|
+
let outSec = ceilFrame(lastWord + tailSec);
|
|
671
|
+
if (after.length > 0) {
|
|
672
|
+
const clear = floorFrame(Math.min(...after.map((w) => w.s)) - EXTRA_CLEARANCE_SEC);
|
|
673
|
+
outSec = Math.min(outSec, Math.max(clear, ceilFrame(lastWord)));
|
|
674
|
+
}
|
|
675
|
+
outSec = Math.min(seg.seconds, outSec);
|
|
676
|
+
if (outSec <= inSec) return;
|
|
677
|
+
const how = [];
|
|
678
|
+
if (inSec > 0) how.push(`lead-in trimmed ${inSec.toFixed(2)}s`);
|
|
679
|
+
if (outSec < seg.seconds) how.push(`tail trimmed ${(seg.seconds - outSec).toFixed(2)}s`);
|
|
680
|
+
if (how.length === 0) return;
|
|
681
|
+
const kept = seg.words.filter((w) => w.s < outSec && w.e > inSec);
|
|
682
|
+
if (before.some((w) => !kept.includes(w))) how.push("extra sound before the line trimmed");
|
|
683
|
+
if (after.some((w) => !kept.includes(w))) how.push("extra sound after the line trimmed");
|
|
684
|
+
seg.inSec = inSec;
|
|
685
|
+
seg.seconds = outSec - inSec;
|
|
686
|
+
seg.words = kept.map((w) => ({ ...w, s: w.s - inSec, e: w.e - inSec }));
|
|
687
|
+
seg.note = withHow(seg.note, how);
|
|
688
|
+
});
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* Where a narrated segment stops speaking, and the air it keeps after (#3384).
|
|
693
|
+
* voiceWords are word times taken from the voice track itself. Without them
|
|
694
|
+
* the end of the voice file stands in for the last word, its own trailing
|
|
695
|
+
* silence counted as part of the air.
|
|
696
|
+
*/
|
|
697
|
+
function narrationEnd(voiceSeconds, voiceWords, endsAd) {
|
|
698
|
+
return {
|
|
699
|
+
spokenEndSec: voiceWords?.length ? Math.max(...voiceWords.map((w) => w.e)) : voiceSeconds,
|
|
700
|
+
airSec: endsAd ? END_AIR_SEC : NARRATION_JOIN_AIR_SEC,
|
|
701
|
+
};
|
|
702
|
+
}
|
|
703
|
+
|
|
704
|
+
const holdFor = (seg) =>
|
|
705
|
+
seg.source === "clip" ? Math.max(0, seg.seconds - Math.max(0, seg.clipSeconds - seg.inSec)) : 0;
|
|
706
|
+
|
|
707
|
+
/**
|
|
708
|
+
* Put a reaction beat into the spine right after its cue line ends, splitting
|
|
709
|
+
* the segment that speaks it when more lines follow in that segment. Returns
|
|
710
|
+
* the scene that speaks the cue, or null when no segment does.
|
|
711
|
+
*/
|
|
712
|
+
function placeReaction(spine, counts, reaction) {
|
|
713
|
+
for (let k = 0; k < spine.length; k++) {
|
|
714
|
+
const seg = spine[k];
|
|
715
|
+
if (seg.reaction) continue;
|
|
716
|
+
const cueEnd = cueEndInSegment(seg, counts, reaction.cueLineId);
|
|
717
|
+
if (cueEnd === null) continue;
|
|
718
|
+
const cut = Math.min(seg.seconds, ceilFrame(cueEnd));
|
|
719
|
+
const beat = {
|
|
720
|
+
sceneIndex: reaction.sceneIndex, source: reaction.source, file: reaction.file,
|
|
721
|
+
clipSeconds: reaction.source === "clip" ? reaction.seconds : null, inSec: 0,
|
|
722
|
+
seconds: Math.min(reaction.seconds, REACTION_MAX_SEC), holdSec: 0,
|
|
723
|
+
audio: reaction.hasAudio ? "clip" : "silence", voice: null, words: null, lineIds: [],
|
|
724
|
+
reaction: { cueLineId: reaction.cueLineId, cueSceneIndex: seg.sceneIndex },
|
|
725
|
+
};
|
|
726
|
+
if (cut >= seg.seconds - 0.05) {
|
|
727
|
+
spine.splice(k + 1, 0, beat);
|
|
728
|
+
return seg.sceneIndex;
|
|
729
|
+
}
|
|
730
|
+
const idx = seg.lineIds.indexOf(reaction.cueLineId);
|
|
731
|
+
const head = {
|
|
732
|
+
...seg, seconds: cut, lineIds: seg.lineIds.slice(0, idx + 1),
|
|
733
|
+
words: seg.words ? seg.words.filter((w) => w.s < cut) : null,
|
|
734
|
+
note: `${seg.note}, until ${reaction.cueLineId} ends at ${cut.toFixed(2)}s`,
|
|
735
|
+
};
|
|
736
|
+
const tail = {
|
|
737
|
+
...seg, inSec: seg.inSec + cut, seconds: seg.seconds - cut, lineIds: seg.lineIds.slice(idx + 1),
|
|
738
|
+
words: seg.words
|
|
739
|
+
? seg.words.filter((w) => w.s >= cut).map((w) => ({ ...w, s: w.s - cut, e: w.e - cut }))
|
|
740
|
+
: null,
|
|
741
|
+
note: `${seg.note}, resumed at ${cut.toFixed(2)}s after reaction ${reaction.sceneIndex}`,
|
|
742
|
+
};
|
|
743
|
+
head.holdSec = holdFor(head);
|
|
744
|
+
tail.holdSec = holdFor(tail);
|
|
745
|
+
spine.splice(k, 1, head, beat, tail);
|
|
746
|
+
return seg.sceneIndex;
|
|
747
|
+
}
|
|
748
|
+
return null;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
/**
|
|
752
|
+
* The sound a reaction beat carries under the next segment (#2406), as a J/L
|
|
753
|
+
* cut: the picture moves on early and the clip's sound from where it stops
|
|
754
|
+
* plays under the next line. A clip that ends with its beat gives up the
|
|
755
|
+
* beat's end, down to REACTION_MIN_BEAT_SEC, rather than replaying sound the
|
|
756
|
+
* beat already played, which was heard as a stutter (#2666, #2700). Shortens
|
|
757
|
+
* the beat, so it runs before the spine's start times are set. Null when
|
|
758
|
+
* under 0.4 s of sound would carry over; the beat fades itself out then.
|
|
759
|
+
*/
|
|
760
|
+
function carveReactionTail(seg, next) {
|
|
761
|
+
const clipLen = seg.clipSeconds ?? seg.seconds;
|
|
762
|
+
const overlap =
|
|
763
|
+
Math.round(Math.min(REACTION_TAIL_SEC, clipLen - seg.inSec - REACTION_MIN_BEAT_SEC, next.seconds) * 1000) / 1000;
|
|
764
|
+
if (overlap < REACTION_FADE_OUT_SEC) return null;
|
|
765
|
+
seg.seconds = Math.min(seg.seconds, Math.round((clipLen - seg.inSec - overlap) * 1000) / 1000);
|
|
766
|
+
return { fromSec: seg.inSec + seg.seconds, seconds: overlap, gain: REACTION_TAIL_GAIN };
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
/**
|
|
770
|
+
* #2666: how long a reaction beat fades in. A generated laugh clip can hum
|
|
771
|
+
* before the laugh starts, so the fade runs across that quiet start, up to its
|
|
772
|
+
* first sound; without the clip's loudness it is the plain edge fade.
|
|
773
|
+
*/
|
|
774
|
+
function quietStart(levels, seconds) {
|
|
775
|
+
const first = levels ? levels.db.findIndex((d) => d >= SOUND_DB) : -1;
|
|
776
|
+
return first > 0 ? Math.min(seconds, Math.max(EDGE_FADE_SEC, first * levels.windowSec)) : EDGE_FADE_SEC;
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
/**
|
|
780
|
+
* Alternate one speaker's scenes in a row between the plain frame and the
|
|
781
|
+
* punch-in, so every cut between them visibly moves in or out. Any other cut
|
|
782
|
+
* starts the alternation again. Marked before reactions split a scene, so both
|
|
783
|
+
* halves keep their scene's frame.
|
|
784
|
+
*/
|
|
785
|
+
function markPunchIns(spine, sameShot) {
|
|
786
|
+
let punched = false;
|
|
787
|
+
spine.forEach((seg, k) => {
|
|
788
|
+
const prev = spine[k - 1];
|
|
789
|
+
punched = Boolean(prev) && sameShot(prev, seg) && !punched;
|
|
790
|
+
if (punched) seg.punchIn = true;
|
|
791
|
+
});
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
const sameSpeaker = (prev, seg) => Boolean(prev.lastSpeaker && seg.firstSpeaker === prev.lastSpeaker);
|
|
795
|
+
|
|
796
|
+
/**
|
|
797
|
+
* The pacings whose edit punches in, each with when a scene plays on in the
|
|
798
|
+
* shot of the one before it. Jump cuts film every scene in one framing. A
|
|
799
|
+
* montage changes shot on every cut, except where the line cut split a long
|
|
800
|
+
* line into pieces that keep their parent's base shot (#3317, #3323).
|
|
801
|
+
*/
|
|
802
|
+
const PUNCH_IN_RUNS = new Map([
|
|
803
|
+
["jump-cuts", sameSpeaker],
|
|
804
|
+
["montage", (prev, seg) =>
|
|
805
|
+
sameSpeaker(prev, seg) && Boolean(seg.baseShotId) && seg.baseShotId === prev.baseShotId],
|
|
806
|
+
]);
|
|
807
|
+
|
|
808
|
+
/**
|
|
809
|
+
* #3365: the scenes the cut cannot be made without, by number. A scene the
|
|
810
|
+
* storyboard plans needs a clip unless --skip names it, even one the pull has
|
|
811
|
+
* no row for. A scene it does not plan is a row a re-board left behind, so it
|
|
812
|
+
* is no loss. With no storyboard, every scene the pull lists counts.
|
|
813
|
+
*/
|
|
814
|
+
export function scenesWithNoClip({ manifest, storyboard, skip, exists = Boolean }) {
|
|
815
|
+
const planned = [...scenePlans(storyboard).keys()];
|
|
816
|
+
const listed = (manifest?.scenes ?? []).map((scene) => scene.sceneIndex);
|
|
817
|
+
const clipped = new Set(
|
|
818
|
+
(manifest?.scenes ?? []).filter((scene) => exists(scene.clip)).map((scene) => scene.sceneIndex),
|
|
819
|
+
);
|
|
820
|
+
return [...new Set(planned.length > 0 ? planned : listed)]
|
|
821
|
+
.filter((n) => !skip.has(n) && !clipped.has(n))
|
|
822
|
+
.sort((a, b) => a - b);
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
export function buildTimeline({
|
|
826
|
+
manifest, storyboard, skip, probe, exists, readWords, readLevels = () => null, stillFill = false,
|
|
827
|
+
}) {
|
|
130
828
|
const plans = scenePlans(storyboard);
|
|
131
829
|
const counts = lineWordCounts(storyboard);
|
|
132
830
|
const spine = [];
|
|
133
831
|
const cutaways = [];
|
|
134
832
|
const waiting = [];
|
|
135
833
|
const dropped = [];
|
|
136
|
-
|
|
834
|
+
const missing = new Set(scenesWithNoClip({ manifest, storyboard, skip, exists }));
|
|
835
|
+
const filled = new Set();
|
|
836
|
+
// A narrated segment is first fitted as a join; the one that ends the ad is
|
|
837
|
+
// refitted once the spine is known.
|
|
838
|
+
const refits = new Map();
|
|
137
839
|
|
|
138
840
|
for (const scene of manifest?.scenes ?? []) {
|
|
139
841
|
const n = scene.sceneIndex;
|
|
140
|
-
const plan = plans.get(n) ?? {
|
|
842
|
+
const plan = plans.get(n) ?? {
|
|
843
|
+
cutaway: false, lineIds: [], cueLineId: null, cutawayType: null, durationSec: null,
|
|
844
|
+
firstSpeaker: null, lastSpeaker: null,
|
|
845
|
+
};
|
|
141
846
|
if (skip.has(n)) {
|
|
142
847
|
dropped.push({ n, reason: `scene ${n}: skipped by --skip` });
|
|
143
848
|
continue;
|
|
@@ -145,17 +850,20 @@ export function buildTimeline({ manifest, storyboard, skip, probe, exists, readW
|
|
|
145
850
|
const nothing = `nothing to cut with (clip ${scene.clipStatus}${scene.error ? `: ${scene.error}` : ""})`;
|
|
146
851
|
|
|
147
852
|
if (plan.cutaway) {
|
|
853
|
+
const cue = { cueLineId: plan.cueLineId, reaction: plan.cutawayType === "reaction" };
|
|
148
854
|
if (exists(scene.clip)) {
|
|
855
|
+
const clip = probe(scene.clip);
|
|
149
856
|
waiting.push({
|
|
150
857
|
sceneIndex: n, source: "clip", file: scene.clip,
|
|
151
|
-
seconds:
|
|
858
|
+
seconds: clip.seconds, hasAudio: clip.hasAudio, ...cue,
|
|
152
859
|
});
|
|
153
|
-
} else if (exists(scene.keyframe)) {
|
|
860
|
+
} else if (stillFill && exists(scene.keyframe)) {
|
|
861
|
+
filled.add(n);
|
|
154
862
|
waiting.push({
|
|
155
863
|
sceneIndex: n, source: "still", file: scene.keyframe,
|
|
156
|
-
seconds: plan.durationSec ?? 4,
|
|
864
|
+
seconds: plan.durationSec ?? 4, hasAudio: false, ...cue,
|
|
157
865
|
});
|
|
158
|
-
} else {
|
|
866
|
+
} else if (!missing.has(n)) {
|
|
159
867
|
dropped.push({ n, reason: `cutaway ${n}: ${nothing}` });
|
|
160
868
|
}
|
|
161
869
|
continue;
|
|
@@ -168,19 +876,31 @@ export function buildTimeline({ manifest, storyboard, skip, probe, exists, readW
|
|
|
168
876
|
if (exists(scene.clip)) {
|
|
169
877
|
const clip = probe(scene.clip);
|
|
170
878
|
const base = {
|
|
171
|
-
sceneIndex: n, source: "clip", file: scene.clip, clipSeconds: clip.seconds,
|
|
172
|
-
|
|
879
|
+
sceneIndex: n, source: "clip", file: scene.clip, clipSeconds: clip.seconds, inSec: 0,
|
|
880
|
+
lineIds: plan.lineIds, firstSpeaker: plan.firstSpeaker, lastSpeaker: plan.lastSpeaker,
|
|
881
|
+
baseShotId: plan.baseShotId ?? null, actionTailSec: plan.actionTailSec ?? 0,
|
|
173
882
|
};
|
|
174
883
|
if (voice && (narrated || !clip.hasAudio)) {
|
|
175
|
-
const
|
|
176
|
-
const
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
884
|
+
const voiceSeconds = probe(voice).seconds;
|
|
885
|
+
const words = readWords(scene.words);
|
|
886
|
+
// A clip's own word times describe the clip, not the voice that replaces its audio.
|
|
887
|
+
const voiceWords = scene.wordsFrom === "voice" ? words : null;
|
|
888
|
+
const fitted = (endsAd) => {
|
|
889
|
+
const fit = fitNarration({
|
|
890
|
+
clipSeconds: clip.seconds, ...narrationEnd(voiceSeconds, voiceWords, endsAd),
|
|
891
|
+
actionTailSec: plan.actionTailSec ?? 0,
|
|
892
|
+
});
|
|
893
|
+
const how = [];
|
|
894
|
+
if (fit.trimmedBy > 0.05) how.push(`clip trimmed ${clip.seconds.toFixed(1)}s -> ${fit.seconds.toFixed(1)}s`);
|
|
895
|
+
if (fit.holdSec > 0.05) how.push(`last frame held ${fit.holdSec.toFixed(1)}s`);
|
|
896
|
+
return {
|
|
897
|
+
seconds: fit.seconds, holdSec: fit.holdSec,
|
|
898
|
+
note: `${flag}, ${voice} replaces the clip's audio${how.length ? ` (${how.join(", ")})` : ""}`,
|
|
899
|
+
};
|
|
900
|
+
};
|
|
901
|
+
const seg = { ...base, audio: "narration", voice, words, ...fitted(false) };
|
|
902
|
+
spine.push(seg);
|
|
903
|
+
refits.set(seg, fitted);
|
|
184
904
|
} else if (narrated) {
|
|
185
905
|
spine.push({
|
|
186
906
|
...base, audio: "silence", voice: null, seconds: clip.seconds, holdSec: 0,
|
|
@@ -198,47 +918,107 @@ export function buildTimeline({ manifest, storyboard, skip, probe, exists, readW
|
|
|
198
918
|
words: null, note: `${flag}, silent clip`,
|
|
199
919
|
});
|
|
200
920
|
}
|
|
201
|
-
startSec += spine[spine.length - 1].seconds;
|
|
202
921
|
continue;
|
|
203
922
|
}
|
|
204
|
-
if (exists(scene.keyframe)) {
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
923
|
+
if (stillFill && exists(scene.keyframe)) {
|
|
924
|
+
filled.add(n);
|
|
925
|
+
const voiceSeconds = voice ? probe(voice).seconds : 0;
|
|
926
|
+
// #1689: on a narrated scene the words are the voice track's, and the
|
|
927
|
+
// voice starts where the segment does, so they need no offset here.
|
|
928
|
+
const words = voice ? readWords(scene.words) : null;
|
|
929
|
+
const voiceWords = scene.wordsFrom === "voice" ? words : null;
|
|
930
|
+
const fitted = (endsAd) => {
|
|
931
|
+
const seconds = voice
|
|
932
|
+
? fitNarration({ clipSeconds: null, ...narrationEnd(voiceSeconds, voiceWords, endsAd) }).seconds
|
|
933
|
+
: plan.durationSec ?? 4;
|
|
934
|
+
return {
|
|
935
|
+
seconds,
|
|
936
|
+
note: `${flag}${voice ? ` + ${voice}` : ", silent"} (${seconds.toFixed(1)}s, clip ${scene.clipStatus})`,
|
|
937
|
+
};
|
|
938
|
+
};
|
|
939
|
+
const seg = {
|
|
940
|
+
sceneIndex: n, source: "still", file: scene.keyframe, clipSeconds: null, inSec: 0,
|
|
941
|
+
audio: voice ? "narration" : "silence", voice, holdSec: 0,
|
|
942
|
+
lineIds: plan.lineIds, firstSpeaker: plan.firstSpeaker, lastSpeaker: plan.lastSpeaker,
|
|
943
|
+
baseShotId: plan.baseShotId ?? null, words, ...fitted(false),
|
|
944
|
+
};
|
|
945
|
+
spine.push(seg);
|
|
946
|
+
if (voice) refits.set(seg, fitted);
|
|
218
947
|
continue;
|
|
219
948
|
}
|
|
220
|
-
dropped.push({ n, reason: `scene ${n}: ${nothing}` });
|
|
949
|
+
if (!missing.has(n)) dropped.push({ n, reason: `scene ${n}: ${nothing}` });
|
|
950
|
+
}
|
|
951
|
+
const unfilled = [...missing].filter((n) => !filled.has(n));
|
|
952
|
+
if (unfilled.length > 0) {
|
|
953
|
+
const listed = new Map((manifest?.scenes ?? []).map((scene) => [scene.sceneIndex, scene]));
|
|
954
|
+
throw new FirstCutError(1, noClipRefusal(unfilled.map((n) => {
|
|
955
|
+
const scene = listed.get(n);
|
|
956
|
+
return {
|
|
957
|
+
n,
|
|
958
|
+
keyframe: Boolean(scene) && exists(scene.keyframe),
|
|
959
|
+
status: scene ? `clip ${scene.clipStatus}${scene.error ? `: ${scene.error}` : ""}` : "clip missing",
|
|
960
|
+
};
|
|
961
|
+
})));
|
|
962
|
+
}
|
|
963
|
+
|
|
964
|
+
const sameShot = PUNCH_IN_RUNS.get(manifest?.shots?.pacing);
|
|
965
|
+
if (sameShot) markPunchIns(spine, sameShot);
|
|
966
|
+
|
|
967
|
+
const noCue = (c) =>
|
|
968
|
+
c.cueLineId ? `${c.cueLineId} is spoken by no scene in the cut` : "the storyboard names no cue line";
|
|
969
|
+
|
|
970
|
+
// Sentence gaps and then joins are closed first so a reaction beat lands
|
|
971
|
+
// right after its cue line ends in the trimmed clip. Reactions change the
|
|
972
|
+
// spine, so they go in before any overlay is placed on it.
|
|
973
|
+
const scriptLines = new Map((storyboard?.script ?? []).map((line) => [line?.lineId, line]));
|
|
974
|
+
const reactionCues = new Set(waiting.filter((w) => w.reaction).map((w) => w.cueLineId));
|
|
975
|
+
const last = spine.at(-1);
|
|
976
|
+
if (refits.has(last) && !reactionCues.has(last.lineIds.at(-1))) Object.assign(last, refits.get(last)(true));
|
|
977
|
+
tightenSentenceGaps(spine, reactionCues, scriptLines, readLevels);
|
|
978
|
+
trimJoins(spine, reactionCues, scriptLines);
|
|
979
|
+
for (const c of waiting.filter((w) => w.reaction)) {
|
|
980
|
+
if (!c.cueLineId || placeReaction(spine, counts, c) === null) {
|
|
981
|
+
dropped.push({ n: c.sceneIndex, reason: `cutaway ${c.sceneIndex}: ${noCue(c)}` });
|
|
982
|
+
}
|
|
221
983
|
}
|
|
984
|
+
spine.forEach((seg, k) => {
|
|
985
|
+
if (!seg.reaction || seg.audio !== "clip") return;
|
|
986
|
+
const tail = spine[k + 1] ? carveReactionTail(seg, spine[k + 1]) : null;
|
|
987
|
+
if (tail) seg.reaction.tail = tail;
|
|
988
|
+
else seg.fadeOutSec = Math.min(REACTION_FADE_OUT_SEC, seg.seconds);
|
|
989
|
+
seg.fadeInSec = quietStart(readLevels(seg.file), seg.seconds);
|
|
990
|
+
});
|
|
991
|
+
let total = 0;
|
|
992
|
+
for (const seg of spine) {
|
|
993
|
+
seg.startSec = total;
|
|
994
|
+
total += seg.seconds;
|
|
995
|
+
}
|
|
996
|
+
spine.forEach((seg, k) => {
|
|
997
|
+
if (seg.reaction?.tail) seg.reaction.tail.atSec = spine[k + 1].startSec;
|
|
998
|
+
});
|
|
222
999
|
|
|
223
|
-
for (const c of waiting) {
|
|
1000
|
+
for (const c of waiting.filter((w) => !w.reaction)) {
|
|
224
1001
|
const cue = c.cueLineId ? cueOffsetOnSpine(spine, counts, c.cueLineId) : null;
|
|
225
1002
|
if (!cue) {
|
|
226
|
-
|
|
227
|
-
? `${c.cueLineId} is spoken by no scene in the cut`
|
|
228
|
-
: "the storyboard names no cue line";
|
|
229
|
-
dropped.push({ n: c.sceneIndex, reason: `cutaway ${c.sceneIndex}: ${why}` });
|
|
1003
|
+
dropped.push({ n: c.sceneIndex, reason: `cutaway ${c.sceneIndex}: ${noCue(c)}` });
|
|
230
1004
|
continue;
|
|
231
1005
|
}
|
|
232
1006
|
cutaways.push({ ...c, startSec: cue.startSec, method: cue.method, spineSceneIndex: cue.spineSceneIndex });
|
|
233
1007
|
}
|
|
234
1008
|
cutaways.sort((a, b) => a.startSec - b.startSec);
|
|
1009
|
+
const beats = spine.filter((seg) => seg.reaction);
|
|
235
1010
|
for (let i = 0; i < cutaways.length; i++) {
|
|
236
1011
|
const c = cutaways[i];
|
|
1012
|
+
const stops = [];
|
|
237
1013
|
const next = cutaways[i + 1];
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
1014
|
+
if (next) stops.push({ at: next.startSec, by: `cutaway ${next.sceneIndex} starts` });
|
|
1015
|
+
const beat = beats.find((b) => b.startSec >= c.startSec);
|
|
1016
|
+
if (beat) stops.push({ at: beat.startSec, by: `reaction ${beat.sceneIndex} starts` });
|
|
1017
|
+
stops.push({ at: total, by: "the ad ends" });
|
|
1018
|
+
const stop = stops.reduce((a, b) => (b.at < a.at ? b : a));
|
|
1019
|
+
if (c.startSec + c.seconds > stop.at) {
|
|
1020
|
+
c.seconds = Math.max(0, stop.at - c.startSec);
|
|
1021
|
+
c.cutShortBy = stop.by;
|
|
242
1022
|
}
|
|
243
1023
|
}
|
|
244
1024
|
|
|
@@ -249,6 +1029,20 @@ export function buildTimeline({ manifest, storyboard, skip, probe, exists, readW
|
|
|
249
1029
|
};
|
|
250
1030
|
}
|
|
251
1031
|
|
|
1032
|
+
/** #3365: why the cut stopped on scenes with no clip, and the person's three ways on. */
|
|
1033
|
+
function noClipRefusal(missing) {
|
|
1034
|
+
const scenes = missing.map((m) => `scene ${m.n} (${m.status})`).join(", ");
|
|
1035
|
+
const skip = `--skip ${missing.map((m) => m.n).join(",")}`;
|
|
1036
|
+
const bare = missing.filter((m) => !m.keyframe).map((m) => m.n);
|
|
1037
|
+
const ways = bare.length === missing.length
|
|
1038
|
+
? `Wait for the clips and pull again, or leave them out with ${skip}.`
|
|
1039
|
+
: `Wait for the clips and pull again, leave them out with ${skip}, ` +
|
|
1040
|
+
"or hold each one's keyframe on screen instead with --still-fill (under its voice track if it has one, " +
|
|
1041
|
+
"else silent for its planned length)." +
|
|
1042
|
+
(bare.length > 0 ? ` --still-fill cannot cover scene ${bare.join(", ")}, which has no keyframe either.` : "");
|
|
1043
|
+
return `No clip for ${scenes}. ${ways}`;
|
|
1044
|
+
}
|
|
1045
|
+
|
|
252
1046
|
function sec(n) {
|
|
253
1047
|
return (Number.isFinite(n) ? Math.max(0, n) : 0).toFixed(3);
|
|
254
1048
|
}
|
|
@@ -262,6 +1056,7 @@ export function buildFfmpegArgs(timeline, { resolve, music, out }) {
|
|
|
262
1056
|
};
|
|
263
1057
|
const filters = [];
|
|
264
1058
|
const pairs = [];
|
|
1059
|
+
const tails = [];
|
|
265
1060
|
|
|
266
1061
|
for (const seg of timeline.spine) {
|
|
267
1062
|
const i =
|
|
@@ -269,26 +1064,77 @@ export function buildFfmpegArgs(timeline, { resolve, music, out }) {
|
|
|
269
1064
|
? addInput("-i", resolve(seg.file))
|
|
270
1065
|
: addInput("-loop", "1", "-t", String(seg.seconds), "-i", resolve(seg.file));
|
|
271
1066
|
const fit = [];
|
|
272
|
-
|
|
273
|
-
|
|
1067
|
+
const inSec = seg.inSec ?? 0;
|
|
1068
|
+
// #2544: sentence gaps come out first, the picture by frame number and the
|
|
1069
|
+
// sound by the same times, so everything after works on the cut timeline.
|
|
1070
|
+
const cuts = seg.cuts ?? [];
|
|
1071
|
+
let heard = `[${i}:a]${stereo},`;
|
|
1072
|
+
if (cuts.length > 0) {
|
|
1073
|
+
const frames = cuts.map((c) => `between(n\\,${Math.round(c.s * FPS)}\\,${Math.round(c.e * FPS) - 1})`);
|
|
1074
|
+
fit.push(`select=not(${frames.join("+")}),setpts=N/FRAME_RATE/TB`);
|
|
1075
|
+
const kept = [0, ...cuts.flatMap((c) => [c.s, c.e])];
|
|
1076
|
+
const pieces = cuts.length + 1;
|
|
1077
|
+
filters.push(`[${i}:a]${stereo},asplit=${pieces}${Array.from({ length: pieces }, (_, p) => `[ap${i}_${p}]`).join("")}`);
|
|
1078
|
+
for (let p = 0; p < pieces; p++) {
|
|
1079
|
+
const end = p < cuts.length ? `:end=${kept[2 * p + 1].toFixed(6)}` : "";
|
|
1080
|
+
filters.push(`[ap${i}_${p}]atrim=start=${kept[2 * p].toFixed(6)}${end},asetpts=PTS-STARTPTS[aq${i}_${p}]`);
|
|
1081
|
+
}
|
|
1082
|
+
filters.push(
|
|
1083
|
+
`${Array.from({ length: pieces }, (_, p) => `[aq${i}_${p}]`).join("")}concat=n=${pieces}:v=0:a=1[ac${i}]`,
|
|
1084
|
+
);
|
|
1085
|
+
heard = `[ac${i}]`;
|
|
1086
|
+
}
|
|
1087
|
+
if (seg.source === "clip" && seg.clipSeconds !== null) {
|
|
1088
|
+
const shown = seg.seconds - seg.holdSec;
|
|
1089
|
+
if (inSec > 0.001 || shown < seg.clipSeconds - 0.01) {
|
|
1090
|
+
// By frame number: a time printed to the millisecond can round past the
|
|
1091
|
+
// frame it names and drop it.
|
|
1092
|
+
const from = Math.round(inSec * FPS);
|
|
1093
|
+
fit.push(`trim=start_frame=${from}:end_frame=${Math.round((inSec + shown) * FPS)},setpts=PTS-STARTPTS`);
|
|
1094
|
+
}
|
|
274
1095
|
}
|
|
275
1096
|
if (seg.holdSec > 0.01) fit.push(`tpad=stop_mode=clone:stop_duration=${sec(seg.holdSec)}`);
|
|
1097
|
+
if (seg.punchIn) fit.push(PUNCH_IN);
|
|
276
1098
|
filters.push(`${norm(`${i}:v`)}${fit.length ? `,${fit.join(",")}` : ""}[v${i}]`);
|
|
1099
|
+
// A segment split around a reaction beat starts partway into its sound too.
|
|
1100
|
+
const skipIn = inSec > 0.001 ? `atrim=start=${sec(inSec)},asetpts=PTS-STARTPTS,` : "";
|
|
1101
|
+
const [fadeIn, fadeOut] = [seg.fadeInSec ?? EDGE_FADE_SEC, seg.fadeOutSec ?? EDGE_FADE_SEC];
|
|
1102
|
+
// A cubic rise keeps a reaction's quiet start near silent until its laugh (#2666).
|
|
1103
|
+
const rise = fadeIn > EDGE_FADE_SEC ? ":curve=cub" : "";
|
|
1104
|
+
const fitLength =
|
|
1105
|
+
`apad,atrim=0:${sec(seg.seconds)},afade=t=in:d=${sec(fadeIn)}${rise},` +
|
|
1106
|
+
`afade=t=out:st=${sec(seg.seconds - fadeOut)}:d=${sec(fadeOut)}`;
|
|
1107
|
+
const fitSound = `${LOUDNORM},${fitLength}`;
|
|
277
1108
|
if (seg.audio === "narration") {
|
|
278
1109
|
const v = addInput("-i", resolve(seg.voice));
|
|
279
|
-
filters.push(`[${v}:a]${stereo},${
|
|
1110
|
+
filters.push(`[${v}:a]${stereo},${skipIn}${fitSound}[a${i}]`);
|
|
1111
|
+
} else if (seg.audio === "clip" && seg.reaction?.tail) {
|
|
1112
|
+
// The tail is cut from the same loudness-normalized sound as the beat,
|
|
1113
|
+
// ducked, faded out, and laid in at the join; the next line's own sound
|
|
1114
|
+
// plays at full level over it (#2406).
|
|
1115
|
+
const t = seg.reaction.tail;
|
|
1116
|
+
filters.push(`${heard}${skipIn}${LOUDNORM},asplit=2[as${i}][at${i}]`);
|
|
1117
|
+
filters.push(`[as${i}]${fitLength}[a${i}]`);
|
|
1118
|
+
filters.push(
|
|
1119
|
+
`[at${i}]atrim=start=${sec(t.fromSec - inSec)}:duration=${sec(t.seconds)},asetpts=PTS-STARTPTS,` +
|
|
1120
|
+
`volume=${t.gain},afade=t=in:d=${EDGE_FADE_SEC},afade=t=out:st=0:d=${sec(t.seconds)},` +
|
|
1121
|
+
`adelay=delays=${Math.round(t.atSec * 1000)}:all=1[tail${i}]`,
|
|
1122
|
+
);
|
|
1123
|
+
tails.push(`[tail${i}]`);
|
|
280
1124
|
} else if (seg.audio === "clip") {
|
|
281
|
-
filters.push(
|
|
1125
|
+
filters.push(`${heard}${skipIn}${fitSound}[a${i}]`);
|
|
282
1126
|
} else {
|
|
283
1127
|
filters.push(`anullsrc=r=48000:cl=stereo,atrim=0:${sec(seg.seconds)}[a${i}]`);
|
|
284
1128
|
}
|
|
285
1129
|
pairs.push(`[v${i}][a${i}]`);
|
|
286
1130
|
}
|
|
287
1131
|
|
|
288
|
-
// A clip whose audio ends before its video would otherwise end the whole cut
|
|
289
|
-
// there under -shortest; padding lets the video decide the length.
|
|
290
1132
|
filters.push(`${pairs.join("")}concat=n=${pairs.length}:v=1:a=1[vcat][voicecat]`);
|
|
291
|
-
|
|
1133
|
+
// The sound is padded or cut to the picture's exact length, so every stream
|
|
1134
|
+
// ends on its own. No -shortest: ffmpeg 8 held the whole ad's frames for it,
|
|
1135
|
+
// and Debian's ffmpeg 5.1 (the worker's) never ended at all (#2988).
|
|
1136
|
+
const pictureSec = timeline.spine.reduce((sum, seg) => sum + seg.seconds, 0);
|
|
1137
|
+
filters.push(`[voicecat]apad,atrim=end=${pictureSec.toFixed(6)}[voice]`);
|
|
292
1138
|
|
|
293
1139
|
let videoLabel = "vcat";
|
|
294
1140
|
timeline.cutaways.forEach((c, j) => {
|
|
@@ -308,7 +1154,7 @@ export function buildFfmpegArgs(timeline, { resolve, music, out }) {
|
|
|
308
1154
|
});
|
|
309
1155
|
|
|
310
1156
|
filters.push(`${ROOM_TONE}[room]`);
|
|
311
|
-
const beds = ["[room]"];
|
|
1157
|
+
const beds = ["[room]", ...tails];
|
|
312
1158
|
if (music) {
|
|
313
1159
|
const m = addInput("-stream_loop", "-1", "-i", music);
|
|
314
1160
|
filters.push(`[${m}:a]${stereo},volume=${MUSIC_VOLUME}[bed]`);
|
|
@@ -326,13 +1172,41 @@ export function buildFfmpegArgs(timeline, { resolve, music, out }) {
|
|
|
326
1172
|
"-map", `[${videoLabel}]`, "-map", `[${audioLabel}]`,
|
|
327
1173
|
"-c:v", "libx264", "-preset", "medium", "-crf", "20", "-r", String(FPS),
|
|
328
1174
|
"-c:a", "aac", "-b:a", "192k",
|
|
329
|
-
"-movflags", "+faststart",
|
|
1175
|
+
"-movflags", "+faststart",
|
|
330
1176
|
out,
|
|
331
1177
|
];
|
|
332
1178
|
}
|
|
333
1179
|
|
|
1180
|
+
const roundMs = (n) => Math.round(n * 1000) / 1000;
|
|
1181
|
+
|
|
1182
|
+
/**
|
|
1183
|
+
* #3300: every stretch of the cut where a scene is the picture, in order. A
|
|
1184
|
+
* cutaway covers the whole frame, so the spine scene under it is off screen
|
|
1185
|
+
* there and its placement splits around it. A scene split around a reaction
|
|
1186
|
+
* beat, or shown twice, has one entry per stretch.
|
|
1187
|
+
*/
|
|
1188
|
+
export function sceneTimes(timeline) {
|
|
1189
|
+
const covers = timeline.cutaways
|
|
1190
|
+
.filter((c) => c.seconds > 0)
|
|
1191
|
+
.map((c) => ({ from: c.startSec, to: c.startSec + c.seconds }));
|
|
1192
|
+
const visible = timeline.spine.flatMap((seg) => {
|
|
1193
|
+
let pieces = [{ from: seg.startSec, to: seg.startSec + seg.seconds }];
|
|
1194
|
+
for (const c of covers) {
|
|
1195
|
+
pieces = pieces.flatMap((p) =>
|
|
1196
|
+
c.to <= p.from || c.from >= p.to
|
|
1197
|
+
? [p]
|
|
1198
|
+
: [{ from: p.from, to: c.from }, { from: c.to, to: p.to }].filter((q) => q.to > q.from),
|
|
1199
|
+
);
|
|
1200
|
+
}
|
|
1201
|
+
return pieces.map((p) => ({ sceneIndex: seg.sceneIndex, startSec: p.from, seconds: p.to - p.from }));
|
|
1202
|
+
});
|
|
1203
|
+
return [...visible, ...timeline.cutaways]
|
|
1204
|
+
.map((p) => ({ scene: p.sceneIndex, startSec: roundMs(p.startSec), seconds: roundMs(p.seconds) }))
|
|
1205
|
+
.filter((p) => p.seconds > 0)
|
|
1206
|
+
.sort((a, b) => a.startSec - b.startSec);
|
|
1207
|
+
}
|
|
1208
|
+
|
|
334
1209
|
export function wordRows(timeline) {
|
|
335
|
-
const ms = (n) => Math.round(n * 1000) / 1000;
|
|
336
1210
|
const rows = [];
|
|
337
1211
|
for (const seg of timeline.spine) {
|
|
338
1212
|
for (const w of seg.words ?? []) {
|
|
@@ -340,7 +1214,7 @@ export function wordRows(timeline) {
|
|
|
340
1214
|
const start = seg.startSec + w.s;
|
|
341
1215
|
const end = seg.startSec + w.e;
|
|
342
1216
|
if (text && end > start) {
|
|
343
|
-
rows.push({ text, start:
|
|
1217
|
+
rows.push({ text, start: roundMs(start), end: roundMs(end), scene: seg.sceneIndex });
|
|
344
1218
|
}
|
|
345
1219
|
}
|
|
346
1220
|
}
|
|
@@ -391,13 +1265,34 @@ export function toSrt(rows) {
|
|
|
391
1265
|
export function sidecarPaths(outPath) {
|
|
392
1266
|
const dir = path.dirname(outPath);
|
|
393
1267
|
const base = path.basename(outPath, path.extname(outPath));
|
|
394
|
-
return {
|
|
1268
|
+
return {
|
|
1269
|
+
words: path.join(dir, `${base}.words.json`),
|
|
1270
|
+
srt: path.join(dir, `${base}.srt`),
|
|
1271
|
+
scenes: path.join(dir, `${base}.scenes.json`),
|
|
1272
|
+
};
|
|
395
1273
|
}
|
|
396
1274
|
|
|
397
1275
|
export function inTheCut(timeline) {
|
|
398
1276
|
const rows = [];
|
|
399
1277
|
for (const seg of timeline.spine) {
|
|
400
1278
|
const label = seg.source === "clip" ? seg.file : `still ${seg.file}`;
|
|
1279
|
+
if (seg.reaction) {
|
|
1280
|
+
const sound = seg.audio === "clip" ? "its own sound" : "silent: the clip has no sound";
|
|
1281
|
+
const tail = seg.reaction.tail;
|
|
1282
|
+
const fadeIn =
|
|
1283
|
+
seg.fadeInSec > EDGE_FADE_SEC ? `, fading in over its quiet first ${seg.fadeInSec.toFixed(2)}s` : "";
|
|
1284
|
+
const fadeOut = tail
|
|
1285
|
+
? `, fading out under the next line at ${tail.atSec.toFixed(2)}s over ${tail.seconds.toFixed(2)}s`
|
|
1286
|
+
: seg.fadeOutSec ? `, fading out over its last ${seg.fadeOutSec.toFixed(2)}s` : "";
|
|
1287
|
+
const fades = fadeIn + fadeOut;
|
|
1288
|
+
rows.push({
|
|
1289
|
+
n: seg.sceneIndex,
|
|
1290
|
+
line:
|
|
1291
|
+
`reaction ${seg.sceneIndex}: ${label} for ${seg.seconds.toFixed(1)}s after ` +
|
|
1292
|
+
`${seg.reaction.cueLineId} in scene ${seg.reaction.cueSceneIndex}, ${sound}${fades}`,
|
|
1293
|
+
});
|
|
1294
|
+
continue;
|
|
1295
|
+
}
|
|
401
1296
|
rows.push({ n: seg.sceneIndex, line: `scene ${seg.sceneIndex}: ${label}${seg.note}` });
|
|
402
1297
|
}
|
|
403
1298
|
for (const c of timeline.cutaways) {
|
|
@@ -417,10 +1312,112 @@ export function inTheCut(timeline) {
|
|
|
417
1312
|
return rows.sort((a, b) => a.n - b.n).map((r) => r.line);
|
|
418
1313
|
}
|
|
419
1314
|
|
|
1315
|
+
/**
|
|
1316
|
+
* #2666: a generated clip can hum or click where nobody speaks, and the cut
|
|
1317
|
+
* plays that stretch with nothing over it. A hum is HUM_BAND's share of the
|
|
1318
|
+
* sound (the 1 kHz whine heard before a first word) holding for HUM_MIN_SEC
|
|
1319
|
+
* over HUM_FLOOR_DB; a click is a sound of at most CLICK_MAX_WINDOWS with
|
|
1320
|
+
* quiet on both sides, loud as the gap cutter reads a sound in near silence.
|
|
1321
|
+
*/
|
|
1322
|
+
export const HUM_BAND = "highpass=f=800,highpass=f=800,lowpass=f=1500,lowpass=f=1500";
|
|
1323
|
+
// White noise measures about -13.5 dB through HUM_BAND at 16 kHz (-10.6 dB on
|
|
1324
|
+
// paper), and the four hums #2666 was calibrated on held -6 to -10 dB.
|
|
1325
|
+
const HUM_SHARE_DB = -10;
|
|
1326
|
+
const HUM_FLOOR_DB = -70;
|
|
1327
|
+
const HUM_MIN_SEC = 0.16;
|
|
1328
|
+
const CLICK_MAX_WINDOWS = 3;
|
|
1329
|
+
|
|
1330
|
+
/**
|
|
1331
|
+
* Where the cut plays a clip's sound with no word in it, in the clip's own
|
|
1332
|
+
* time: before a dialogue clip's first word, after its last, and a reaction's
|
|
1333
|
+
* quiet start (under its cubic fade-in when the cut fades it).
|
|
1334
|
+
*/
|
|
1335
|
+
function edgeStretches(seg, levels) {
|
|
1336
|
+
const inSec = seg.inSec ?? 0;
|
|
1337
|
+
if (seg.reaction) {
|
|
1338
|
+
const quiet = quietStart(levels, seg.seconds);
|
|
1339
|
+
if (quiet <= EDGE_FADE_SEC) return [];
|
|
1340
|
+
const fadeIn = seg.fadeInSec ?? 0;
|
|
1341
|
+
return [{ side: "before its laugh", from: inSec, to: inSec + quiet, at: seg.startSec, fadeIn }];
|
|
1342
|
+
}
|
|
1343
|
+
if (!seg.words?.length) return [];
|
|
1344
|
+
// inSec and the words are on the clip's timeline with its sentence gaps
|
|
1345
|
+
// taken out; the levels are the file's own, so each edge goes back through
|
|
1346
|
+
// the cuts before it.
|
|
1347
|
+
const raw = (t) => {
|
|
1348
|
+
let at = t;
|
|
1349
|
+
for (const c of seg.cuts ?? []) if (c.s <= at) at += c.e - c.s;
|
|
1350
|
+
return at;
|
|
1351
|
+
};
|
|
1352
|
+
const first = seg.words[0].s;
|
|
1353
|
+
const last = Math.max(...seg.words.map((w) => w.e));
|
|
1354
|
+
const shown = seg.seconds - (seg.holdSec ?? 0);
|
|
1355
|
+
return [
|
|
1356
|
+
{ side: "before its first word", from: raw(inSec), to: raw(inSec + first), at: seg.startSec, fadeIn: 0 },
|
|
1357
|
+
{
|
|
1358
|
+
side: "after its last word", from: raw(inSec + last), to: raw(inSec + shown),
|
|
1359
|
+
at: seg.startSec + last, fadeIn: 0,
|
|
1360
|
+
},
|
|
1361
|
+
];
|
|
1362
|
+
}
|
|
1363
|
+
|
|
1364
|
+
/**
|
|
1365
|
+
* The hums and clicks the cut plays at the edges of its clips, one line each,
|
|
1366
|
+
* for the caller to listen to. It never changes the cut. readEdgeLevels(file)
|
|
1367
|
+
* gives { windowSec, db, bandDb } from time 0, or null.
|
|
1368
|
+
*/
|
|
1369
|
+
export function edgeSounds(timeline, readEdgeLevels) {
|
|
1370
|
+
const found = [];
|
|
1371
|
+
for (const seg of timeline.spine) {
|
|
1372
|
+
if (seg.source !== "clip" || seg.audio !== "clip") continue;
|
|
1373
|
+
const levels = readEdgeLevels(seg.file);
|
|
1374
|
+
if (!levels) continue;
|
|
1375
|
+
const { windowSec, db, bandDb } = levels;
|
|
1376
|
+
for (const stretch of edgeStretches(seg, levels)) {
|
|
1377
|
+
const from = Math.ceil(stretch.from / windowSec - 1e-9);
|
|
1378
|
+
const to = Math.min(db.length, Math.floor(stretch.to / windowSec + 1e-9));
|
|
1379
|
+
if (to - from < 2) continue;
|
|
1380
|
+
const heard = [];
|
|
1381
|
+
for (let i = from; i < to; i++) {
|
|
1382
|
+
const into = (i + 0.5) * windowSec - stretch.from;
|
|
1383
|
+
const faded = stretch.fadeIn > 0 && into < stretch.fadeIn;
|
|
1384
|
+
const gain = faded ? 60 * Math.log10(Math.max(1e-3, into / stretch.fadeIn)) : 0;
|
|
1385
|
+
heard.push({ i, db: db[i] + gain, share: bandDb[i] - db[i] });
|
|
1386
|
+
}
|
|
1387
|
+
const cutAt = (i) => stretch.at + i * windowSec - stretch.from;
|
|
1388
|
+
const where = `scene ${seg.sceneIndex}, ${stretch.side}`;
|
|
1389
|
+
let run = 0;
|
|
1390
|
+
let longest = { length: 0, end: 0 };
|
|
1391
|
+
heard.forEach((w, k) => {
|
|
1392
|
+
run = w.db >= HUM_FLOOR_DB && w.share >= HUM_SHARE_DB ? run + 1 : 0;
|
|
1393
|
+
if (run > longest.length) longest = { length: run, end: k };
|
|
1394
|
+
});
|
|
1395
|
+
if (longest.length * windowSec >= HUM_MIN_SEC - 1e-9) {
|
|
1396
|
+
const start = heard[longest.end - longest.length + 1].i;
|
|
1397
|
+
found.push(
|
|
1398
|
+
`${where}: a hum for ${(longest.length * windowSec).toFixed(2)}s at ${cutAt(start).toFixed(2)}s in the cut`,
|
|
1399
|
+
);
|
|
1400
|
+
}
|
|
1401
|
+
const sorted = heard.map((w) => w.db).sort((a, b) => a - b);
|
|
1402
|
+
const loud = Math.max(FAINT_SOUND_DB, sorted[sorted.length >> 1] + SOUND_OVER_FLOOR_DB);
|
|
1403
|
+
for (let k = 1; k < heard.length - 1; k++) {
|
|
1404
|
+
if (heard[k].db < loud || heard[k - 1].db >= QUIET_DB) continue;
|
|
1405
|
+
let end = k;
|
|
1406
|
+
while (end < heard.length && heard[end].db >= loud) end++;
|
|
1407
|
+
if (end < heard.length && heard[end].db < QUIET_DB && end - k <= CLICK_MAX_WINDOWS) {
|
|
1408
|
+
found.push(`${where}: a click at ${cutAt(heard[k].i).toFixed(2)}s in the cut`);
|
|
1409
|
+
}
|
|
1410
|
+
k = end;
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
return found;
|
|
1415
|
+
}
|
|
1416
|
+
|
|
420
1417
|
export function noWordTimes(timeline) {
|
|
421
1418
|
const rows = timeline.spine
|
|
422
1419
|
.filter((seg) => !seg.words || seg.words.length === 0)
|
|
423
|
-
.map((seg) => `scene ${seg.sceneIndex} (no words.json)`);
|
|
1420
|
+
.map((seg) => `scene ${seg.sceneIndex} (${seg.reaction ? "reaction" : "no words.json"})`);
|
|
424
1421
|
for (const c of timeline.cutaways) rows.push(`scene ${c.sceneIndex} (cutaway)`);
|
|
425
1422
|
return rows;
|
|
426
1423
|
}
|
|
@@ -430,13 +1427,23 @@ function fail(code, message) {
|
|
|
430
1427
|
process.exit(code);
|
|
431
1428
|
}
|
|
432
1429
|
|
|
1430
|
+
/** A refusal the CLI prints to stderr and exits with `code`, so another caller can do the same or not. */
|
|
1431
|
+
export class FirstCutError extends Error {
|
|
1432
|
+
constructor(code, message) {
|
|
1433
|
+
super(message);
|
|
1434
|
+
this.name = "FirstCutError";
|
|
1435
|
+
this.code = code;
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
|
|
433
1439
|
function parseArgs(argv) {
|
|
434
|
-
const opts = { dir: undefined, out: undefined, skip: new Set(), music: true };
|
|
1440
|
+
const opts = { dir: undefined, out: undefined, skip: new Set(), music: true, stillFill: false };
|
|
435
1441
|
for (let i = 0; i < argv.length; i++) {
|
|
436
1442
|
const a = argv[i];
|
|
437
1443
|
if (a === "--out") opts.out = argv[++i];
|
|
438
1444
|
else if (a === "--skip") opts.skip = new Set((argv[++i] ?? "").split(",").filter(Boolean).map(Number));
|
|
439
1445
|
else if (a === "--no-music") opts.music = false;
|
|
1446
|
+
else if (a === "--still-fill") opts.stillFill = true;
|
|
440
1447
|
else if (a.startsWith("--")) fail(2, `unknown flag ${a}\n${USAGE}`);
|
|
441
1448
|
else if (opts.dir === undefined) opts.dir = a;
|
|
442
1449
|
else fail(2, `unexpected argument ${a}\n${USAGE}`);
|
|
@@ -445,26 +1452,29 @@ function parseArgs(argv) {
|
|
|
445
1452
|
return opts;
|
|
446
1453
|
}
|
|
447
1454
|
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
1455
|
+
/**
|
|
1456
|
+
* The whole cut, as the CLI makes it, for any caller (#2988: the worker cuts a
|
|
1457
|
+
* Video-page run with this). Renders `out` (default <dir>/cut.mp4) and its word
|
|
1458
|
+
* sidecars, and throws FirstCutError where the CLI would exit. The CLI's report
|
|
1459
|
+
* is printed by main() from what this returns.
|
|
1460
|
+
*/
|
|
1461
|
+
export function firstCut({ dir, out: outOpt, skip = new Set(), music: musicOn = true, stillFill = false }) {
|
|
451
1462
|
for (const bin of ["ffmpeg", "ffprobe"]) {
|
|
452
1463
|
if (spawnSync(bin, ["-version"]).status !== 0) {
|
|
453
|
-
|
|
1464
|
+
throw new FirstCutError(2, `${bin} is not installed. Install ffmpeg (brew install ffmpeg / apt install ffmpeg).`);
|
|
454
1465
|
}
|
|
455
1466
|
}
|
|
456
1467
|
const encoders = spawnSync("ffmpeg", ["-hide_banner", "-encoders"]).stdout?.toString() ?? "";
|
|
457
1468
|
if (!/\blibx264\b/.test(encoders)) {
|
|
458
|
-
|
|
1469
|
+
throw new FirstCutError(2, "This ffmpeg has no libx264 encoder (H.264). Install a full ffmpeg build: brew install ffmpeg / apt install ffmpeg.");
|
|
459
1470
|
}
|
|
460
1471
|
|
|
461
|
-
const dir = opts.dir;
|
|
462
1472
|
const manifestPath = path.join(dir, "manifest.json");
|
|
463
1473
|
if (!existsSync(manifestPath)) {
|
|
464
|
-
|
|
1474
|
+
throw new FirstCutError(2, `${manifestPath} not found. Run: exodus video pull <runId> --out ${dir}`);
|
|
465
1475
|
}
|
|
466
1476
|
const manifest = JSON.parse(readFileSync(manifestPath, "utf8"));
|
|
467
|
-
const out = path.resolve(
|
|
1477
|
+
const out = path.resolve(outOpt ?? path.join(dir, "cut.mp4"));
|
|
468
1478
|
const here = (file) => path.join(dir, file);
|
|
469
1479
|
const exists = (file) => Boolean(file) && existsSync(here(file));
|
|
470
1480
|
|
|
@@ -478,11 +1488,14 @@ function main() {
|
|
|
478
1488
|
};
|
|
479
1489
|
const probe = (file) => {
|
|
480
1490
|
const raw = execFileSync("ffprobe", [
|
|
481
|
-
"-v", "error", "-show_entries", "stream=codec_type:format=duration", "-of", "json", here(file),
|
|
1491
|
+
"-v", "error", "-show_entries", "stream=codec_type,duration:stream_disposition=attached_pic:format=duration", "-of", "json", here(file),
|
|
482
1492
|
]).toString();
|
|
483
1493
|
const info = JSON.parse(raw);
|
|
1494
|
+
// A clip's container runs a few ms past its picture on the audio's padding;
|
|
1495
|
+
// the picture is what the cut shows, so its length is the clip's length.
|
|
1496
|
+
const picture = Number((info.streams ?? []).find((s) => s.codec_type === "video" && !s.disposition?.attached_pic)?.duration);
|
|
484
1497
|
return {
|
|
485
|
-
seconds: Number(info.format?.duration ?? 0),
|
|
1498
|
+
seconds: picture > 0 ? picture : Number(info.format?.duration ?? 0),
|
|
486
1499
|
hasAudio: (info.streams ?? []).some((s) => s.codec_type === "audio"),
|
|
487
1500
|
};
|
|
488
1501
|
};
|
|
@@ -494,19 +1507,21 @@ function main() {
|
|
|
494
1507
|
const timeline = buildTimeline({
|
|
495
1508
|
manifest,
|
|
496
1509
|
storyboard,
|
|
497
|
-
skip
|
|
1510
|
+
skip,
|
|
1511
|
+
stillFill,
|
|
498
1512
|
probe,
|
|
499
1513
|
exists,
|
|
500
1514
|
readWords: (file) => {
|
|
501
1515
|
const rows = readJson(file);
|
|
502
1516
|
return Array.isArray(rows) && rows.length > 0 ? rows : null;
|
|
503
1517
|
},
|
|
1518
|
+
readLevels: (file) => clipLevels(here(file)),
|
|
504
1519
|
});
|
|
505
1520
|
if (timeline.spine.length === 0) {
|
|
506
|
-
|
|
1521
|
+
throw new FirstCutError(1, `No scene has a clip or a keyframe. Check: exodus video status ${manifest.runId}`);
|
|
507
1522
|
}
|
|
508
1523
|
|
|
509
|
-
const music =
|
|
1524
|
+
const music = musicOn && exists(manifest.music) ? manifest.music : null;
|
|
510
1525
|
const ffmpegArgs = buildFfmpegArgs(timeline, {
|
|
511
1526
|
resolve: here,
|
|
512
1527
|
music: music ? here(music) : null,
|
|
@@ -515,7 +1530,7 @@ function main() {
|
|
|
515
1530
|
const res = spawnSync("ffmpeg", ffmpegArgs, { stdio: "inherit" });
|
|
516
1531
|
if (res.status !== 0) {
|
|
517
1532
|
const quoted = ffmpegArgs.map((a) => (/[\s\[\];]/.test(a) ? `'${a}'` : a)).join(" ");
|
|
518
|
-
|
|
1533
|
+
throw new FirstCutError(1, `ffmpeg failed. The command it ran:\n ffmpeg ${quoted}`);
|
|
519
1534
|
}
|
|
520
1535
|
|
|
521
1536
|
const rows = wordRows(timeline);
|
|
@@ -527,11 +1542,27 @@ function main() {
|
|
|
527
1542
|
rmSync(sidecars.words, { force: true });
|
|
528
1543
|
rmSync(sidecars.srt, { force: true });
|
|
529
1544
|
}
|
|
1545
|
+
writeFileSync(sidecars.scenes, `${JSON.stringify(sceneTimes(timeline), null, 2)}\n`);
|
|
530
1546
|
|
|
531
1547
|
const duration = execFileSync("ffprobe", [
|
|
532
1548
|
"-v", "error", "-show_entries", "format=duration", "-of", "default=nw=1:nk=1", out,
|
|
533
1549
|
]).toString().trim();
|
|
534
|
-
|
|
1550
|
+
return { out, durationSec: Number(duration), manifest, timeline, music, rows, sidecars };
|
|
1551
|
+
}
|
|
1552
|
+
|
|
1553
|
+
function main() {
|
|
1554
|
+
const opts = parseArgs(process.argv.slice(2));
|
|
1555
|
+
let cut;
|
|
1556
|
+
try {
|
|
1557
|
+
cut = firstCut(opts);
|
|
1558
|
+
} catch (e) {
|
|
1559
|
+
if (e instanceof FirstCutError) fail(e.code, e.message);
|
|
1560
|
+
throw e;
|
|
1561
|
+
}
|
|
1562
|
+
const { out, durationSec, manifest, timeline, music, rows, sidecars } = cut;
|
|
1563
|
+
const here = (file) => path.join(opts.dir, file);
|
|
1564
|
+
|
|
1565
|
+
console.log(`Wrote ${out} (${durationSec.toFixed(1)}s, ${WIDTH}x${HEIGHT})`);
|
|
535
1566
|
console.log("In the cut:");
|
|
536
1567
|
for (const line of inTheCut(timeline)) console.log(" " + line);
|
|
537
1568
|
if (timeline.left.length > 0) {
|
|
@@ -548,8 +1579,25 @@ function main() {
|
|
|
548
1579
|
}
|
|
549
1580
|
const silent = noWordTimes(timeline);
|
|
550
1581
|
if (silent.length > 0) console.log(`No word times: ${silent.join(", ")}`);
|
|
1582
|
+
const edges = edgeSounds(timeline, (file) => {
|
|
1583
|
+
const levels = clipLevels(here(file));
|
|
1584
|
+
const band = levels && clipLevels(here(file), HUM_BAND);
|
|
1585
|
+
return band ? { ...levels, bandDb: band.db } : null;
|
|
1586
|
+
});
|
|
1587
|
+
if (edges.length > 0) {
|
|
1588
|
+
console.log("Sound at the edges (listen before uploading; the cut is unchanged):");
|
|
1589
|
+
for (const line of edges) console.log(" " + line);
|
|
1590
|
+
}
|
|
551
1591
|
console.log(`Room tone: under the whole ad. Music bed: ${music ?? "none"}`);
|
|
552
1592
|
console.log(`Upload it: exodus video upload ${manifest.runId} --file ${out}`);
|
|
553
1593
|
}
|
|
554
1594
|
|
|
555
|
-
|
|
1595
|
+
// #2988: the basename test too, because a bundler that inlines this file into
|
|
1596
|
+
// its entry would otherwise run the CLI inside whatever imported it.
|
|
1597
|
+
if (
|
|
1598
|
+
process.argv[1] &&
|
|
1599
|
+
path.basename(process.argv[1]) === "first-cut.mjs" &&
|
|
1600
|
+
import.meta.url === pathToFileURL(process.argv[1]).href
|
|
1601
|
+
) {
|
|
1602
|
+
main();
|
|
1603
|
+
}
|