@softure-ai/marketing-kit 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +71 -0
- package/README.md +66 -9
- package/dist/cli/footage.js +1 -1
- package/dist/cli/footage.js.map +1 -1
- package/dist/compose/timeline.d.ts +23 -3
- package/dist/compose/timeline.d.ts.map +1 -1
- package/dist/compose/timeline.js +31 -11
- package/dist/compose/timeline.js.map +1 -1
- package/dist/config/actions-schema.d.ts +14 -14
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +2 -0
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +83 -70
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +21 -1
- package/dist/config/schema.js.map +1 -1
- package/dist/config/shot-steps.d.ts +16 -16
- package/dist/footage/captions.d.ts +45 -3
- package/dist/footage/captions.d.ts.map +1 -1
- package/dist/footage/captions.js +65 -5
- package/dist/footage/captions.js.map +1 -1
- package/dist/footage/compose.d.ts.map +1 -1
- package/dist/footage/compose.js +8 -4
- package/dist/footage/compose.js.map +1 -1
- package/dist/footage/declick.d.ts +106 -0
- package/dist/footage/declick.d.ts.map +1 -0
- package/dist/footage/declick.js +296 -0
- package/dist/footage/declick.js.map +1 -0
- package/dist/footage/draft.d.ts +8 -3
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +55 -22
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/edl.d.ts +28 -2
- package/dist/footage/edl.d.ts.map +1 -1
- package/dist/footage/edl.js +53 -2
- package/dist/footage/edl.js.map +1 -1
- package/dist/footage/film.d.ts +29 -3
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js +12 -0
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/isolate-ranges.d.ts +37 -0
- package/dist/footage/isolate-ranges.d.ts.map +1 -0
- package/dist/footage/isolate-ranges.js +113 -0
- package/dist/footage/isolate-ranges.js.map +1 -0
- package/dist/footage/loudness.d.ts +58 -0
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +80 -0
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/paid.d.ts +18 -0
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +32 -2
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +43 -5
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +103 -20
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/providers.d.ts +5 -0
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/publish.d.ts +11 -2
- package/dist/footage/publish.d.ts.map +1 -1
- package/dist/footage/publish.js +14 -10
- package/dist/footage/publish.js.map +1 -1
- package/dist/footage/speech-ranges.d.ts +65 -0
- package/dist/footage/speech-ranges.d.ts.map +1 -0
- package/dist/footage/speech-ranges.js +98 -0
- package/dist/footage/speech-ranges.js.map +1 -0
- package/dist/index.d.ts +10 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -7
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/schema/edl.schema.json +17 -0
- package/schema/marketing.schema.json +43 -1
- package/src/cli/footage.ts +1 -1
- package/src/compose/timeline.ts +41 -10
- package/src/config/config.ts +2 -0
- package/src/config/schema.ts +23 -1
- package/src/footage/captions.ts +81 -6
- package/src/footage/compose.ts +8 -4
- package/src/footage/declick.ts +353 -0
- package/src/footage/draft.ts +58 -23
- package/src/footage/edl.ts +69 -2
- package/src/footage/film.ts +37 -3
- package/src/footage/isolate-ranges.ts +161 -0
- package/src/footage/loudness.ts +112 -0
- package/src/footage/paid.ts +40 -2
- package/src/footage/pipeline.ts +155 -22
- package/src/footage/providers.ts +2 -0
- package/src/footage/publish.ts +20 -11
- package/src/footage/speech-ranges.ts +131 -0
- package/src/index.ts +70 -0
package/src/footage/draft.ts
CHANGED
|
@@ -2,15 +2,16 @@ import { copyFileSync, existsSync, mkdirSync, rmSync, statSync, writeFileSync }
|
|
|
2
2
|
import { basename, dirname, join, resolve } from "node:path";
|
|
3
3
|
import { fileURLToPath } from "node:url";
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import type { CaptionLayout, LayoutOverride } from "../compose/timeline.js";
|
|
6
6
|
import type { BrandColors } from "../config/colors.js";
|
|
7
7
|
import { err, ok } from "../voice/provider.js";
|
|
8
8
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
9
|
-
import { getCaptionSpans, getFootageCaptionLayout,
|
|
9
|
+
import { getCaptionChunkLayouts, getCaptionSpans, getFootageCaptionChunks, getFootageCaptionLayout, isSameCaptionPlace, type CaptionWordsFilm } from "./captions.js";
|
|
10
10
|
import { getSegmentScales, getSegmentZoomEases } from "./compose.js";
|
|
11
11
|
import { chooseFootageDraftFont } from "./draft-font.js";
|
|
12
12
|
import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
|
|
13
|
-
import {
|
|
13
|
+
import { declickSpeech, describeDeclick, type DeclickReport } from "./declick.js";
|
|
14
|
+
import { getDeclickLevel, getFootageEdit, type FootageFilm } from "./film.js";
|
|
14
15
|
import { duckingKeyframes, getDuckingOptions } from "./music.js";
|
|
15
16
|
import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
|
|
16
17
|
import { findFootageMusic, getFfmpegReporting, getFrame, type FootageBrand, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
|
|
@@ -80,16 +81,25 @@ export function toAssColor(hex: string): string {
|
|
|
80
81
|
return `&H${alpha}${pair(4)}${pair(2)}${pair(0)}`;
|
|
81
82
|
}
|
|
82
83
|
|
|
83
|
-
/**
|
|
84
|
+
/**
|
|
85
|
+
* Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. A
|
|
86
|
+
* no-break space (U+00A0) becomes ASS's hard space `\h`, which libass never breaks a line at, as the render's HTML.
|
|
87
|
+
*/
|
|
84
88
|
function toAssText(text: string): string {
|
|
85
|
-
return text
|
|
89
|
+
return text
|
|
90
|
+
.replace(/\\/g, "/")
|
|
91
|
+
.replace(/\{/g, "(")
|
|
92
|
+
.replace(/\}/g, ")")
|
|
93
|
+
.replace(/[^\S\u00a0]+/g, " ")
|
|
94
|
+
.replace(/^ +| +$/g, "")
|
|
95
|
+
.replace(/\u00a0/g, "\\h");
|
|
86
96
|
}
|
|
87
97
|
|
|
88
98
|
/** The caption colours the draft paints with (`BrandColors`). */
|
|
89
99
|
export type FootageDraftCaptionColors = Pick<BrandColors, "captionBackground" | "captionText" | "captionHighlight">;
|
|
90
100
|
|
|
91
101
|
export interface FootageDraftCaptionsInput {
|
|
92
|
-
film: Pick<FootageFilm, "captions" | "captionStyle"
|
|
102
|
+
film: Pick<FootageFilm, "captions" | "captionStyle"> & CaptionWordsFilm;
|
|
93
103
|
edl: Edl;
|
|
94
104
|
/** The film's words in film time (`remapWords`). */
|
|
95
105
|
words: TimedWord[];
|
|
@@ -107,7 +117,7 @@ export interface FootageDraftCaptionsInput {
|
|
|
107
117
|
* A `highlight` chunk as ASS events: its span cut wherever a word lights up or goes out, each piece showing the chunk
|
|
108
118
|
* with the words lit then in `highlight` (a word stays lit for the render's 0.32 s from its start).
|
|
109
119
|
*/
|
|
110
|
-
function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: number; end: number }, colors: { text: string; highlight: string }): string[] {
|
|
120
|
+
function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: number; end: number }, colors: { text: string; highlight: string }, styleName: string): string[] {
|
|
111
121
|
const cs = (seconds: number): number => Math.round(seconds * 100);
|
|
112
122
|
const [from, to] = [cs(span.start), cs(span.end)];
|
|
113
123
|
const lights = chunk.words.map((word) => ({ on: cs(word.start), off: cs(word.start + HIGHLIGHT_SECONDS) }));
|
|
@@ -122,7 +132,7 @@ function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: numb
|
|
|
122
132
|
return light.on <= at && at < light.off ? `{\\1c${colors.highlight}&}${shown}{\\1c${colors.text}&}` : shown;
|
|
123
133
|
})
|
|
124
134
|
.join(" ");
|
|
125
|
-
events.push(`Dialogue: 0,${toAssTime(at / 100)},${toAssTime(next / 100)}
|
|
135
|
+
events.push(`Dialogue: 0,${toAssTime(at / 100)},${toAssTime(next / 100)},${styleName},,0,0,0,,${text}`);
|
|
126
136
|
});
|
|
127
137
|
return events;
|
|
128
138
|
}
|
|
@@ -132,12 +142,13 @@ function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: numb
|
|
|
132
142
|
* caption box from its top (or from its foot, with `bottom`). `phrase` (the default): the film's phrases
|
|
133
143
|
* (`phraseChunks`), white bold text with a dark outline and shadow. `highlight`: the render's chunks of up to four
|
|
134
144
|
* words (`captionChunks`) on an opaque box of `captionBackground`, in `captionText`, the word being said in
|
|
135
|
-
* `captionHighlight`.
|
|
145
|
+
* `captionHighlight`. Both keep the film's kept words with the next, as the render does. Null when the film has no
|
|
146
|
+
* captions or nothing is said.
|
|
136
147
|
*/
|
|
137
148
|
export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): string | null {
|
|
138
149
|
if (!input.film.captions) return null;
|
|
139
150
|
const isHighlight = input.film.captionStyle === "highlight";
|
|
140
|
-
const chunks =
|
|
151
|
+
const chunks = getFootageCaptionChunks(input.film, input.words);
|
|
141
152
|
if (chunks.length === 0) return null;
|
|
142
153
|
const spans = getCaptionSpans(chunks, getEdlDuration(input.edl));
|
|
143
154
|
const { caption, frame } = input;
|
|
@@ -156,8 +167,8 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
|
|
|
156
167
|
outline: Math.max(1, Math.round(caption.fontSize / 16)),
|
|
157
168
|
shadow: Math.max(1, Math.round(caption.fontSize / 25)),
|
|
158
169
|
};
|
|
159
|
-
const
|
|
160
|
-
|
|
170
|
+
const toStyle = (name: string, place: CaptionLayout): string => [
|
|
171
|
+
name,
|
|
161
172
|
family,
|
|
162
173
|
caption.fontSize,
|
|
163
174
|
look.primary,
|
|
@@ -176,18 +187,31 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
|
|
|
176
187
|
look.outline,
|
|
177
188
|
look.shadow,
|
|
178
189
|
// ASS alignment 2 hangs the text from MarginV above the bottom, 8 from MarginV below the top.
|
|
179
|
-
|
|
180
|
-
Math.round(
|
|
181
|
-
Math.round(
|
|
182
|
-
Math.round(
|
|
190
|
+
place.bottom === undefined ? 8 : 2,
|
|
191
|
+
Math.round(place.left),
|
|
192
|
+
Math.round(place.right),
|
|
193
|
+
Math.round(place.bottom ?? place.top),
|
|
183
194
|
1,
|
|
184
195
|
].join(",");
|
|
196
|
+
// A chunk in a segment's own box gets a style of that box (Caption2, Caption3, ...); the film's box stays `Caption`.
|
|
197
|
+
const styles = [toStyle("Caption", caption)];
|
|
198
|
+
const moved: CaptionLayout[] = [];
|
|
199
|
+
const styleNames = getCaptionChunkLayouts(chunks, input.edl, caption).map((place) => {
|
|
200
|
+
if (isSameCaptionPlace(place, caption)) return "Caption";
|
|
201
|
+
let index = moved.findIndex((other) => isSameCaptionPlace(other, place));
|
|
202
|
+
if (index === -1) {
|
|
203
|
+
index = moved.push(place) - 1;
|
|
204
|
+
styles.push(toStyle(`Caption${index + 2}`, place));
|
|
205
|
+
}
|
|
206
|
+
return `Caption${index + 2}`;
|
|
207
|
+
});
|
|
185
208
|
const highlight = { text: text.replace(/^&H../, "&H"), highlight: toAssColor(colors.captionHighlight).replace(/^&H../, "&H") };
|
|
186
209
|
const events = chunks.flatMap((chunk, index) => {
|
|
187
210
|
const span = spans[index] as { start: number; end: number };
|
|
188
|
-
|
|
211
|
+
const styleName = styleNames[index] ?? "Caption";
|
|
212
|
+
if (isHighlight) return buildHighlightEvents(chunk, span, highlight, styleName);
|
|
189
213
|
const line = toAssText(chunk.words.map((word) => word.text).join(" "));
|
|
190
|
-
return [`Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)}
|
|
214
|
+
return [`Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},${styleName},,0,0,0,,${line}`];
|
|
191
215
|
});
|
|
192
216
|
return [
|
|
193
217
|
"[Script Info]",
|
|
@@ -199,7 +223,7 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
|
|
|
199
223
|
"",
|
|
200
224
|
"[V4+ Styles]",
|
|
201
225
|
"Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding",
|
|
202
|
-
`Style: ${style}
|
|
226
|
+
...styles.map((style) => `Style: ${style}`),
|
|
203
227
|
"",
|
|
204
228
|
"[Events]",
|
|
205
229
|
"Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text",
|
|
@@ -427,6 +451,8 @@ export interface FootageDraft {
|
|
|
427
451
|
width: number;
|
|
428
452
|
height: number;
|
|
429
453
|
bytes: number;
|
|
454
|
+
/** What the speech track's declick stage did, as the render does it; null when the film's declick is off. */
|
|
455
|
+
declick: DeclickReport | null;
|
|
430
456
|
}
|
|
431
457
|
|
|
432
458
|
/** The draft's music track: a file to duck, or why there is none. */
|
|
@@ -478,13 +504,16 @@ async function runRenderFootageDraft(
|
|
|
478
504
|
const draftDir = resolve(options.buildDir, "draft");
|
|
479
505
|
rmSync(draftDir, { recursive: true, force: true });
|
|
480
506
|
mkdirSync(draftDir, { recursive: true });
|
|
481
|
-
const
|
|
507
|
+
const declickLevel = getDeclickLevel(film);
|
|
508
|
+
const items = ["speech", ...(declickLevel === "off" ? [] : ["declick"]), ...(music.value.kind === "file" ? ["music"] : []), "film"];
|
|
482
509
|
const report = (item: string, done: number): void => {
|
|
483
510
|
const itemIndex = items.indexOf(item) + 1;
|
|
484
511
|
options.onProgress?.({ step: "draft", item, itemIndex, itemCount: items.length, done: Math.min(seconds * items.length, (itemIndex - 1) * seconds + done), total: seconds * items.length });
|
|
485
512
|
};
|
|
486
513
|
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
|
|
487
|
-
const
|
|
514
|
+
const cutSpeech = join(draftDir, "speech.mp3");
|
|
515
|
+
// The declicked track stays WAV: the draft's one encode is its only lossy step after the cut.
|
|
516
|
+
const speech = declickLevel === "off" ? cutSpeech : join(draftDir, "speech.declicked.wav");
|
|
488
517
|
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
489
518
|
const frame = getFrame(film);
|
|
490
519
|
const brandFont = chooseFootageDraftFont(options.brand?.fonts.body ?? null);
|
|
@@ -509,7 +538,13 @@ async function runRenderFootageDraft(
|
|
|
509
538
|
const output = resolve(options.output);
|
|
510
539
|
mkdirSync(dirname(output), { recursive: true });
|
|
511
540
|
report("speech", 0);
|
|
512
|
-
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech,
|
|
541
|
+
await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, cutSpeech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: cutSpeech, onSeconds: (done) => report("speech", done) });
|
|
542
|
+
let declick: DeclickReport | null = null;
|
|
543
|
+
if (declickLevel !== "off") {
|
|
544
|
+
report("declick", 0);
|
|
545
|
+
declick = await declickSpeech({ input: cutSpeech, output: speech, words, level: declickLevel, seconds, ...aborting, onSeconds: (done) => report("declick", done) });
|
|
546
|
+
log(describeDeclick(declick));
|
|
547
|
+
}
|
|
513
548
|
let musicFile: string | null = null;
|
|
514
549
|
if (music.value.kind === "file") {
|
|
515
550
|
musicFile = join(draftDir, "music.mp3");
|
|
@@ -542,5 +577,5 @@ async function runRenderFootageDraft(
|
|
|
542
577
|
const draftMusic: FootageDraftMusic = music.value.kind === "file" ? "played" : music.value.kind;
|
|
543
578
|
const notes = [captions === null ? "no captions" : `captions in ${font.family}`, draftMusic === "played" ? "music" : draftMusic === "none" ? "no music" : "music not cached"];
|
|
544
579
|
log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes, ${notes.join(", ")}.`);
|
|
545
|
-
return ok({ output, music: draftMusic, captionFont: captions === null ? null : font.family, seconds, width: draft.width, height: draft.height, bytes });
|
|
580
|
+
return ok({ output, music: draftMusic, captionFont: captions === null ? null : font.family, seconds, width: draft.width, height: draft.height, bytes, declick });
|
|
546
581
|
}
|
package/src/footage/edl.ts
CHANGED
|
@@ -27,6 +27,16 @@ export interface EdlAudio {
|
|
|
27
27
|
ownVolume: number;
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
/**
|
|
31
|
+
* A segment's own caption box, merged over the film's (`getFootageCaptionLayout`) for the captions that start in it.
|
|
32
|
+
* Px of the film's frame, as `CaptionLayout`: `bottom` anchors the box by its foot (above the frame's foot) and wins;
|
|
33
|
+
* `top` alone hangs the box from that top.
|
|
34
|
+
*/
|
|
35
|
+
export interface EdlCaption {
|
|
36
|
+
bottom?: number;
|
|
37
|
+
top?: number;
|
|
38
|
+
}
|
|
39
|
+
|
|
30
40
|
export interface EdlSegment {
|
|
31
41
|
/** Unique in the cut: `s<n>` (the sentence it starts with), `r<n>` (b-roll), `w<n>` (a talking clip without words). */
|
|
32
42
|
id: string;
|
|
@@ -39,6 +49,8 @@ export interface EdlSegment {
|
|
|
39
49
|
/** `kenburns`: a slow zoom over a segment without speech. */
|
|
40
50
|
motion: SegmentMotion;
|
|
41
51
|
audio?: EdlAudio;
|
|
52
|
+
/** Moves the caption box for the captions that start in this segment; absent: the film's box. */
|
|
53
|
+
caption?: EdlCaption;
|
|
42
54
|
}
|
|
43
55
|
|
|
44
56
|
export interface Edl {
|
|
@@ -75,6 +87,13 @@ export const edlSchema = z
|
|
|
75
87
|
})
|
|
76
88
|
.optional()
|
|
77
89
|
.describe("Sound from another clip or span: a J- or L-cut, or b-roll over speech. Without it, the segment plays its own sound."),
|
|
90
|
+
caption: z
|
|
91
|
+
.strictObject({
|
|
92
|
+
bottom: z.number().min(0).optional().describe("The caption box's foot, in px above the frame's foot; anchors the box by its foot and wins over top."),
|
|
93
|
+
top: z.number().min(0).optional().describe("The caption box's top, in px below the frame's top; used when bottom is absent."),
|
|
94
|
+
})
|
|
95
|
+
.optional()
|
|
96
|
+
.describe("Moves the caption box for the captions that start in this segment (px of the film's frame). Without it, the film's caption box."),
|
|
78
97
|
}),
|
|
79
98
|
)
|
|
80
99
|
.min(1, "the cut needs at least one segment")
|
|
@@ -221,11 +240,18 @@ export function getSoundSpan(segment: EdlSegment): { clip: string; start: number
|
|
|
221
240
|
return segment.audio ?? { clip: segment.clip, start: segment.start, end: segment.end };
|
|
222
241
|
}
|
|
223
242
|
|
|
243
|
+
/** The frame a segment's caption box must stay in: its height and the height of a one-line caption card, in px. */
|
|
244
|
+
export interface EdlCaptionBounds {
|
|
245
|
+
frameHeight: number;
|
|
246
|
+
cardHeight: number;
|
|
247
|
+
}
|
|
248
|
+
|
|
224
249
|
/**
|
|
225
250
|
* What makes an EDL (a hand-edited one above all) unusable with these clips; empty when it is fine: another film,
|
|
226
|
-
* an unknown clip, a span past a clip's end, a too short segment, a repeated id, a sound span of another length
|
|
251
|
+
* an unknown clip, a span past a clip's end, a too short segment, a repeated id, a sound span of another length, and
|
|
252
|
+
* (with `captionBounds`) a segment's caption box that would leave the frame.
|
|
227
253
|
*/
|
|
228
|
-
export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[], filmId: string): string[] {
|
|
254
|
+
export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[], filmId: string, captionBounds?: EdlCaptionBounds): string[] {
|
|
229
255
|
const problems: string[] = [];
|
|
230
256
|
if (edl.film !== filmId) problems.push(`it belongs to film "${edl.film}", not "${filmId}"`);
|
|
231
257
|
const durations = new Map(clips.map((clip) => [clip.id, clip.duration]));
|
|
@@ -241,6 +267,7 @@ export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[],
|
|
|
241
267
|
seen.add(segment.id);
|
|
242
268
|
checkSpan(at, segment);
|
|
243
269
|
if (segment.end - segment.start < MIN_SEGMENT_SECONDS) problems.push(`${at}: shorter than ${MIN_SEGMENT_SECONDS} s (start ${segment.start}, end ${segment.end})`);
|
|
270
|
+
if (captionBounds !== undefined) problems.push(...findCaptionProblems(`${at}.caption`, segment.caption, captionBounds));
|
|
244
271
|
const { audio } = segment;
|
|
245
272
|
if (audio === undefined) return;
|
|
246
273
|
checkSpan(`${at}.audio`, audio);
|
|
@@ -251,6 +278,19 @@ export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[],
|
|
|
251
278
|
return problems;
|
|
252
279
|
}
|
|
253
280
|
|
|
281
|
+
/** A caption box's edges past the frame: each edge at most the frame's height minus one card, so a card always fits. */
|
|
282
|
+
function findCaptionProblems(at: string, caption: EdlCaption | undefined, bounds: EdlCaptionBounds): string[] {
|
|
283
|
+
const limit = Math.round(bounds.frameHeight - bounds.cardHeight);
|
|
284
|
+
const problems: string[] = [];
|
|
285
|
+
for (const edge of ["bottom", "top"] as const) {
|
|
286
|
+
const value = caption?.[edge];
|
|
287
|
+
if (value === undefined) continue;
|
|
288
|
+
if (value < 0) problems.push(`${at}.${edge}: ${value} px is outside the frame`);
|
|
289
|
+
else if (value > limit) problems.push(`${at}.${edge}: ${value} px leaves no room for a caption card (at most ${limit} px in a ${bounds.frameHeight} px frame)`);
|
|
290
|
+
}
|
|
291
|
+
return problems;
|
|
292
|
+
}
|
|
293
|
+
|
|
254
294
|
/** `validateEdl` under its first name. */
|
|
255
295
|
export const findEdlProblems = validateEdl;
|
|
256
296
|
|
|
@@ -326,6 +366,11 @@ export function diffEdl(before: Edl, after: Edl): string[] {
|
|
|
326
366
|
if (a.transition !== b.transition) lines.push(`${id}: ${a.transition} → ${b.transition}`);
|
|
327
367
|
if (a.motion !== b.motion) lines.push(`${id}: motion ${a.motion} → ${b.motion}`);
|
|
328
368
|
}
|
|
369
|
+
for (const segment of after.segments) {
|
|
370
|
+
const previous = old.get(segment.id);
|
|
371
|
+
if (previous === undefined || isSameCaption(previous.caption, segment.caption)) continue;
|
|
372
|
+
lines.push(isMovedCaption(segment.caption) ? `${segment.id} captions moved` : `${segment.id} captions back in the film's box`);
|
|
373
|
+
}
|
|
329
374
|
for (const segment of after.segments) {
|
|
330
375
|
const was = old.get(segment.id)?.audio;
|
|
331
376
|
const { audio } = segment;
|
|
@@ -341,10 +386,32 @@ export function diffEdl(before: Edl, after: Edl): string[] {
|
|
|
341
386
|
return lines;
|
|
342
387
|
}
|
|
343
388
|
|
|
389
|
+
/** Whether a segment's caption field moves its box (an empty object does not). */
|
|
390
|
+
function isMovedCaption(caption: EdlCaption | undefined): boolean {
|
|
391
|
+
return caption?.bottom !== undefined || caption?.top !== undefined;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function isSameCaption(a: EdlCaption | undefined, b: EdlCaption | undefined): boolean {
|
|
395
|
+
return a?.bottom === b?.bottom && a?.top === b?.top;
|
|
396
|
+
}
|
|
397
|
+
|
|
344
398
|
export function getEdlDuration(edl: Edl): number {
|
|
345
399
|
return r3(edl.segments.reduce((sum, segment) => sum + (segment.end - segment.start), 0));
|
|
346
400
|
}
|
|
347
401
|
|
|
402
|
+
/**
|
|
403
|
+
* The index of the segment that plays at `time` (film seconds): the one whose span holds it; a time before the film
|
|
404
|
+
* gives the first segment, one at or past the cut's end the last.
|
|
405
|
+
*/
|
|
406
|
+
export function findSegmentAt(edl: Edl, time: number): number {
|
|
407
|
+
const starts = getSegmentStarts(edl);
|
|
408
|
+
let found = 0;
|
|
409
|
+
starts.forEach((start, index) => {
|
|
410
|
+
if (start <= time) found = index;
|
|
411
|
+
});
|
|
412
|
+
return found;
|
|
413
|
+
}
|
|
414
|
+
|
|
348
415
|
/** Where each segment starts in the film, in seconds. */
|
|
349
416
|
export function getSegmentStarts(edl: Edl): number[] {
|
|
350
417
|
const starts: number[] = [];
|
package/src/footage/film.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { VideoFormat } from "../compose/timeline.js";
|
|
2
2
|
import type { EndCard } from "../film.js";
|
|
3
|
-
import type {
|
|
3
|
+
import type { DeclickLevel } from "./declick.js";
|
|
4
|
+
import type { LoudnessTarget, MixLoudnessTarget } from "./loudness.js";
|
|
4
5
|
|
|
5
6
|
/**
|
|
6
7
|
* A footage film: filmed clips (a person talking to camera, b-roll) cut by their transcript into a captioned
|
|
@@ -131,18 +132,28 @@ export interface FootageFilm {
|
|
|
131
132
|
* `CAPTION_MIN_FONT_SCALE` of the box's size) until it fits. Absent: no limit. The draft keeps its font size.
|
|
132
133
|
*/
|
|
133
134
|
captionMaxLines?: number;
|
|
135
|
+
/**
|
|
136
|
+
* Words a caption card must not end on: the card hands them to the next one, so `w` stays with `domu`. Compared
|
|
137
|
+
* lower-case; a number always keeps with the word after it. Absent: the language's `DEFAULT_CAPTION_KEEP_WITH_NEXT`;
|
|
138
|
+
* empty: numbers only.
|
|
139
|
+
*/
|
|
140
|
+
captionKeepWithNext?: string[];
|
|
134
141
|
/** The cut's look: zoom size, fade length, Ken Burns, the end card's whoosh. Absent fields take `DEFAULT_EDIT`. */
|
|
135
142
|
edit?: Partial<FootageEdit>;
|
|
136
143
|
/** The closing card after the last segment; null: the film ends with its last segment. */
|
|
137
144
|
endCard: EndCard | null;
|
|
145
|
+
/** The speech track's cleanup before its loudness normalization. Absent: none. */
|
|
146
|
+
speech?: FootageSpeech;
|
|
138
147
|
/** The speech track's loudness, normalized before the music is mixed under it. Absent: −14 LUFS, −1 dBTP; null: left as filmed. */
|
|
139
148
|
loudness?: LoudnessTarget | null;
|
|
140
149
|
/**
|
|
141
150
|
* The finished film's mix (speech, music, sound effects), measured after the render and normalized in two passes of
|
|
142
151
|
* `loudnorm` (one linear gain; the picture is copied, only the sound re-encoded), so the published file plays at the
|
|
143
|
-
* target. A silent mix is left as it is. Absent: −14 LUFS, −1 dBTP; null: the mix as rendered.
|
|
152
|
+
* target. A silent mix is left as it is. Absent: −14 LUFS, −1 dBTP; null: the mix as rendered. `keep: "speech"`
|
|
153
|
+
* applies no gain while the mix measures inside `target ± tolerance` (default 1 LU), so the speech keeps its level
|
|
154
|
+
* whatever the music volume (see `MixLoudnessTarget`).
|
|
144
155
|
*/
|
|
145
|
-
mixLoudness?:
|
|
156
|
+
mixLoudness?: MixLoudnessTarget | null;
|
|
146
157
|
/**
|
|
147
158
|
* The cover JPG written next to the film. Absent or without `at`: the middle of the first talking segment; `at`:
|
|
148
159
|
* that second of the film; null: no cover.
|
|
@@ -150,12 +161,35 @@ export interface FootageFilm {
|
|
|
150
161
|
cover?: { at?: number } | null;
|
|
151
162
|
}
|
|
152
163
|
|
|
164
|
+
/** Settings of the speech track itself. */
|
|
165
|
+
export interface FootageSpeech {
|
|
166
|
+
/**
|
|
167
|
+
* Mouth clicks and lip smacks between words (issue #1084): `gentle` lowers the pauses by 15 dB and removes impulses
|
|
168
|
+
* that jump 15 dB above the band's median, `strong` by 20 dB and 9 dB. Absent or `off`: no pass, the track unchanged.
|
|
169
|
+
*/
|
|
170
|
+
declick?: DeclickLevel;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** The film's declick level; `off` when it sets none. */
|
|
174
|
+
export function getDeclickLevel(film: Pick<FootageFilm, "speech">): DeclickLevel {
|
|
175
|
+
return film.speech?.declick ?? "off";
|
|
176
|
+
}
|
|
177
|
+
|
|
153
178
|
/** Filler words per language, written with repeated letters collapsed (`yyy` and `yyyy` are both `y`). */
|
|
154
179
|
export const DEFAULT_FILLERS: Record<string, string[]> = {
|
|
155
180
|
pl: ["y", "e", "yhm", "yh", "ym", "em", "hm", "mhm", "eh", "uh"],
|
|
156
181
|
en: ["um", "uh", "uhm", "er", "erm", "hm", "mhm", "ah"],
|
|
157
182
|
};
|
|
158
183
|
|
|
184
|
+
/**
|
|
185
|
+
* Words a caption card does not end on, per language: Polish one-letter words and short prepositions, English
|
|
186
|
+
* articles. A language without a list keeps numbers only.
|
|
187
|
+
*/
|
|
188
|
+
export const DEFAULT_CAPTION_KEEP_WITH_NEXT: Record<string, string[]> = {
|
|
189
|
+
pl: ["a", "i", "o", "u", "w", "z", "we", "ze", "na", "do", "od", "po", "za", "ku"],
|
|
190
|
+
en: ["a", "an", "the"],
|
|
191
|
+
};
|
|
192
|
+
|
|
159
193
|
/** The default fillers of a language; none for a language without a list. */
|
|
160
194
|
export function getDefaultFillers(language: string): string[] {
|
|
161
195
|
return DEFAULT_FILLERS[language] ?? [];
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { existsSync, mkdirSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
|
|
4
|
+
import { err, ok } from "../voice/provider.js";
|
|
5
|
+
import { probeAudio } from "./audio.js";
|
|
6
|
+
import { writeCacheFile } from "./cache.js";
|
|
7
|
+
import type { Edl } from "./edl.js";
|
|
8
|
+
import type { FootageFilm } from "./film.js";
|
|
9
|
+
import { getClipCacheKey } from "./paid.js";
|
|
10
|
+
import { getFfmpegReporting, type FootageResult, type FootageRunOptions, type PreparedClip } from "./pipeline.js";
|
|
11
|
+
import { runFfmpeg, withFfmpegFailureReporting } from "./process.js";
|
|
12
|
+
import { toPaidAudio } from "./providers.js";
|
|
13
|
+
import {
|
|
14
|
+
buildRangeAudioArgs,
|
|
15
|
+
buildRangeLayArgs,
|
|
16
|
+
getSpeechRangeCache,
|
|
17
|
+
getSpeechRangeWindows,
|
|
18
|
+
planSpeechRanges,
|
|
19
|
+
type LaidRange,
|
|
20
|
+
type SpeechRange,
|
|
21
|
+
} from "./speech-ranges.js";
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Voice isolation of only what the cut hears (issue #1087): each talking clip's sound ranges (`planSpeechRanges`) are
|
|
25
|
+
* cleaned one by one and cached by the clip's identity, the range in milliseconds and the provider, so an unchanged
|
|
26
|
+
* segment is never paid twice and a changed cut pays only its new ranges. The cleaned ranges are laid back over the
|
|
27
|
+
* clip's own sound into one file as long as the clip: the speech map `renderFootage` takes.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
export type IsolateSpeechRangesOptions = Pick<FootageRunOptions, "buildDir" | "cacheDir" | "providers" | "isCommit" | "signal" | "onCharge" | "onProgress" | "onFfmpegFailure"> & {
|
|
31
|
+
/** Seconds added on each side of a sound span; default `DEFAULT_SPEECH_RANGE_MARGIN` (0.25). */
|
|
32
|
+
margin?: number;
|
|
33
|
+
log?: (line: string) => void;
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
export interface IsolatedSpeechRange extends SpeechRange {
|
|
37
|
+
/** The paid cache's key and file of the range's cleaned voice. */
|
|
38
|
+
cacheKey: string;
|
|
39
|
+
path: string;
|
|
40
|
+
/** Whether the cleaned voice was in the cache before this call. */
|
|
41
|
+
cached: boolean;
|
|
42
|
+
/** Whether it is laid in the clip's speech file (cached, or bought with `isCommit`). */
|
|
43
|
+
laid: boolean;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface IsolatedSpeech {
|
|
47
|
+
/** Every clip's audio for the speech track: its laid file, or the normalized clip when none of its ranges is laid. */
|
|
48
|
+
speech: Record<string, string>;
|
|
49
|
+
ranges: IsolatedSpeechRange[];
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Cleans the voice of the ranges the cut hears and answers the speech map the render takes (`TranscribeOutcome.speech`).
|
|
54
|
+
* With `isCommit` every uncached range is bought (`onCharge` step `isolate`, item the clip id, with its `range`);
|
|
55
|
+
* without it only cached ranges are laid, a free draft, and `ranges` names what is still missing. Each clip with a laid
|
|
56
|
+
* range gets `<buildDir>/audio/<id>.ranges.mp3`.
|
|
57
|
+
*/
|
|
58
|
+
export function isolateSpeechRanges(film: FootageFilm, clips: PreparedClip[], edl: Edl, options: IsolateSpeechRangesOptions): Promise<FootageResult<IsolatedSpeech>> {
|
|
59
|
+
const log = options.log ?? ((): void => undefined);
|
|
60
|
+
return withFfmpegFailureReporting(getFfmpegReporting({ log, ...(options.onFfmpegFailure === undefined ? {} : { onFfmpegFailure: options.onFfmpegFailure }) }), () =>
|
|
61
|
+
runIsolateSpeechRanges(film, clips, edl, options, log),
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** The body of `isolateSpeechRanges`, inside its ffmpeg failure reporting. */
|
|
66
|
+
async function runIsolateSpeechRanges(
|
|
67
|
+
film: FootageFilm,
|
|
68
|
+
clips: PreparedClip[],
|
|
69
|
+
edl: Edl,
|
|
70
|
+
options: IsolateSpeechRangesOptions,
|
|
71
|
+
log: (line: string) => void,
|
|
72
|
+
): Promise<FootageResult<IsolatedSpeech>> {
|
|
73
|
+
const { buildDir, providers, signal } = options;
|
|
74
|
+
const aborting = signal === undefined ? {} : { signal };
|
|
75
|
+
const planned = planSpeechRanges(
|
|
76
|
+
edl,
|
|
77
|
+
clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration })),
|
|
78
|
+
options.margin === undefined ? {} : { margin: options.margin },
|
|
79
|
+
);
|
|
80
|
+
const byClip = new Map(clips.map((clip) => [clip.clip.id, clip]));
|
|
81
|
+
const clipKeys = new Map<string, string>();
|
|
82
|
+
const ranges: IsolatedSpeechRange[] = [];
|
|
83
|
+
for (const range of planned) {
|
|
84
|
+
const clip = byClip.get(range.clip);
|
|
85
|
+
if (clip === undefined) continue;
|
|
86
|
+
let clipKey = clipKeys.get(range.clip);
|
|
87
|
+
if (clipKey === undefined) {
|
|
88
|
+
clipKey = await getClipCacheKey(clip.source, clip.clip);
|
|
89
|
+
clipKeys.set(range.clip, clipKey);
|
|
90
|
+
}
|
|
91
|
+
const cache = getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider: providers.isolation.id });
|
|
92
|
+
const cached = existsSync(cache.path);
|
|
93
|
+
ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached, laid: cached });
|
|
94
|
+
}
|
|
95
|
+
const buying = options.isCommit ? ranges.filter((range) => !range.cached) : [];
|
|
96
|
+
const total = buying.reduce((sum, range) => sum + (range.endMs - range.startMs) / 1000, 0);
|
|
97
|
+
let done = 0;
|
|
98
|
+
const report = (range: SpeechRange, index: number): void => {
|
|
99
|
+
options.onProgress?.({ step: "isolate", item: range.clip, itemIndex: index + 1, itemCount: buying.length, done: Math.round(done * 1000) / 1000, total: Math.round(total * 1000) / 1000 });
|
|
100
|
+
};
|
|
101
|
+
const workDir = join(buildDir, "audio", "ranges");
|
|
102
|
+
for (const [index, range] of buying.entries()) {
|
|
103
|
+
const clip = byClip.get(range.clip);
|
|
104
|
+
if (clip === undefined) continue;
|
|
105
|
+
const units = (range.endMs - range.startMs) / 1000;
|
|
106
|
+
report(range, index);
|
|
107
|
+
mkdirSync(workDir, { recursive: true });
|
|
108
|
+
// The provider sees the clip's id and the range as the file name.
|
|
109
|
+
const input = join(workDir, `${range.clip}.${String(range.startMs)}-${String(range.endMs)}.mp3`);
|
|
110
|
+
await runFfmpeg(`Cutting ${describeRange(range)} for voice isolation`, buildRangeAudioArgs(clip.file, range, input), { ...aborting, output: input });
|
|
111
|
+
signal?.throwIfAborted();
|
|
112
|
+
log(`voice isolation (${providers.isolation.id}): ${describeRange(range)}, ${units.toFixed(1)} s.`);
|
|
113
|
+
const cleaned = await providers.isolation.isolate({ audio: input, ...aborting });
|
|
114
|
+
signal?.throwIfAborted();
|
|
115
|
+
if (!cleaned.ok) return err(`${describeRange(range)}: ${cleaned.error}`);
|
|
116
|
+
const paid = toPaidAudio(cleaned.value);
|
|
117
|
+
writeCacheFile(range.path, paid.audio);
|
|
118
|
+
range.laid = true;
|
|
119
|
+
options.onCharge?.({
|
|
120
|
+
step: "isolate",
|
|
121
|
+
item: range.clip,
|
|
122
|
+
provider: providers.isolation.id,
|
|
123
|
+
units,
|
|
124
|
+
...(paid.credits === undefined ? {} : { credits: paid.credits }),
|
|
125
|
+
cacheKey: range.cacheKey,
|
|
126
|
+
range: { start: range.startMs / 1000, end: range.endMs / 1000 },
|
|
127
|
+
});
|
|
128
|
+
log(`voice isolation: saved ${range.path} (commit it).`);
|
|
129
|
+
done += units;
|
|
130
|
+
report(range, index);
|
|
131
|
+
}
|
|
132
|
+
const missing = ranges.filter((range) => !range.laid);
|
|
133
|
+
if (missing.length > 0) {
|
|
134
|
+
const seconds = missing.reduce((sum, range) => sum + (range.endMs - range.startMs) / 1000, 0);
|
|
135
|
+
log(`voice isolation DRY RUN: ${String(missing.length)} range${missing.length === 1 ? "" : "s"} (${seconds.toFixed(1)} s) not cached, laid with the clip's own sound. Run with --commit to pay.`);
|
|
136
|
+
}
|
|
137
|
+
const speech: Record<string, string> = {};
|
|
138
|
+
for (const clip of clips) {
|
|
139
|
+
const id = clip.clip.id;
|
|
140
|
+
speech[id] = clip.file;
|
|
141
|
+
const laidRanges = ranges.filter((range) => range.clip === id && range.laid);
|
|
142
|
+
if (laidRanges.length === 0) continue;
|
|
143
|
+
const laid: LaidRange[] = [];
|
|
144
|
+
for (const window of getSpeechRangeWindows(laidRanges)) {
|
|
145
|
+
const file = laidRanges.find((range) => range === window.range)?.path;
|
|
146
|
+
if (file === undefined) continue;
|
|
147
|
+
const probed = await probeAudio(file, aborting);
|
|
148
|
+
if (!probed.ok) return err(`${describeRange(window.range)}: the cleaned voice at ${file} does not read (${probed.error}); delete it and run again.`);
|
|
149
|
+
laid.push({ window, file, isMono: probed.probe.channels === 1 });
|
|
150
|
+
}
|
|
151
|
+
const output = join(buildDir, "audio", `${id}.ranges.mp3`);
|
|
152
|
+
mkdirSync(join(buildDir, "audio"), { recursive: true });
|
|
153
|
+
await runFfmpeg(`Laying the cleaned voice over clip ${id}`, buildRangeLayArgs(clip.file, laid, output), { ...aborting, output });
|
|
154
|
+
speech[id] = output;
|
|
155
|
+
}
|
|
156
|
+
return ok({ speech, ranges });
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function describeRange(range: SpeechRange): string {
|
|
160
|
+
return `clip ${range.clip} ${(range.startMs / 1000).toFixed(3)}-${(range.endMs / 1000).toFixed(3)} s`;
|
|
161
|
+
}
|