@softure-ai/marketing-kit 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +98 -0
- package/README.md +130 -17
- package/dist/cli/footage.d.ts.map +1 -1
- package/dist/cli/footage.js +5 -2
- package/dist/cli/footage.js.map +1 -1
- package/dist/compose/compose.d.ts.map +1 -1
- package/dist/compose/compose.js +3 -3
- package/dist/compose/compose.js.map +1 -1
- package/dist/compose/timeline.d.ts +16 -0
- package/dist/compose/timeline.d.ts.map +1 -1
- package/dist/compose/timeline.js +17 -1
- package/dist/compose/timeline.js.map +1 -1
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +25 -2
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +17 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +61 -3
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/audio.d.ts +6 -2
- package/dist/footage/audio.d.ts.map +1 -1
- package/dist/footage/audio.js +15 -3
- package/dist/footage/audio.js.map +1 -1
- package/dist/footage/captions.d.ts +26 -3
- package/dist/footage/captions.d.ts.map +1 -1
- package/dist/footage/captions.js +35 -2
- package/dist/footage/captions.js.map +1 -1
- package/dist/footage/compose.d.ts +13 -2
- package/dist/footage/compose.d.ts.map +1 -1
- package/dist/footage/compose.js +79 -28
- package/dist/footage/compose.js.map +1 -1
- package/dist/footage/cuts.d.ts +44 -0
- package/dist/footage/cuts.d.ts.map +1 -0
- package/dist/footage/cuts.js +83 -0
- package/dist/footage/cuts.js.map +1 -0
- package/dist/footage/disk.d.ts +25 -0
- package/dist/footage/disk.d.ts.map +1 -0
- package/dist/footage/disk.js +72 -0
- package/dist/footage/disk.js.map +1 -0
- package/dist/footage/draft.d.ts +119 -0
- package/dist/footage/draft.d.ts.map +1 -0
- package/dist/footage/draft.js +296 -0
- package/dist/footage/draft.js.map +1 -0
- package/dist/footage/edl.d.ts +2 -1
- package/dist/footage/edl.d.ts.map +1 -1
- package/dist/footage/edl.js +5 -2
- package/dist/footage/edl.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +2 -0
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/film.d.ts +54 -3
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js +7 -1
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/keyterms.d.ts +26 -0
- package/dist/footage/keyterms.d.ts.map +1 -0
- package/dist/footage/keyterms.js +36 -0
- package/dist/footage/keyterms.js.map +1 -0
- package/dist/footage/loudness.d.ts +20 -3
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +36 -18
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/music.d.ts +3 -0
- package/dist/footage/music.d.ts.map +1 -1
- package/dist/footage/music.js +5 -0
- package/dist/footage/music.js.map +1 -1
- package/dist/footage/normalize.d.ts +30 -2
- package/dist/footage/normalize.d.ts.map +1 -1
- package/dist/footage/normalize.js +39 -11
- package/dist/footage/normalize.js.map +1 -1
- package/dist/footage/paid.d.ts +2 -1
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +13 -3
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/picture.d.ts +42 -0
- package/dist/footage/picture.d.ts.map +1 -0
- package/dist/footage/picture.js +141 -0
- package/dist/footage/picture.js.map +1 -0
- package/dist/footage/pipeline.d.ts +66 -4
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +172 -34
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/process.d.ts +2 -0
- package/dist/footage/process.d.ts.map +1 -1
- package/dist/footage/process.js +1 -0
- package/dist/footage/process.js.map +1 -1
- package/dist/footage/providers.d.ts +12 -3
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/publish.d.ts +4 -2
- package/dist/footage/publish.d.ts.map +1 -1
- package/dist/footage/publish.js +8 -5
- package/dist/footage/publish.js.map +1 -1
- package/dist/footage/render.d.ts +18 -3
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +67 -16
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/tools.d.ts +15 -3
- package/dist/footage/tools.d.ts.map +1 -1
- package/dist/footage/tools.js +15 -2
- package/dist/footage/tools.js.map +1 -1
- package/dist/index.d.ts +14 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -9
- package/dist/index.js.map +1 -1
- package/docker/footage-worker.Dockerfile +7 -0
- package/fonts/DejaVuSans-Bold-Latin.ttf +0 -0
- package/fonts/LICENSE-DejaVu.txt +46 -0
- package/fonts/README.md +12 -0
- package/package.json +2 -1
- package/schema/marketing.schema.json +112 -4
- package/src/cli/footage.ts +4 -2
- package/src/compose/compose.ts +13 -3
- package/src/compose/timeline.ts +27 -1
- package/src/config/config.ts +26 -2
- package/src/config/schema.ts +63 -3
- package/src/footage/audio.ts +16 -3
- package/src/footage/captions.ts +45 -4
- package/src/footage/compose.ts +90 -28
- package/src/footage/cuts.ts +104 -0
- package/src/footage/disk.ts +75 -0
- package/src/footage/draft.ts +395 -0
- package/src/footage/edl.ts +5 -2
- package/src/footage/elevenlabs.ts +1 -0
- package/src/footage/film.ts +63 -5
- package/src/footage/keyterms.ts +38 -0
- package/src/footage/loudness.ts +38 -16
- package/src/footage/music.ts +7 -0
- package/src/footage/normalize.ts +51 -10
- package/src/footage/paid.ts +12 -3
- package/src/footage/picture.ts +179 -0
- package/src/footage/pipeline.ts +219 -37
- package/src/footage/process.ts +3 -0
- package/src/footage/providers.ts +10 -3
- package/src/footage/publish.ts +8 -5
- package/src/footage/render.ts +77 -16
- package/src/footage/tools.ts +23 -3
- package/src/index.ts +70 -4
package/src/footage/loudness.ts
CHANGED
|
@@ -8,7 +8,8 @@ import { runFfmpeg, runProcess } from "./process.js";
|
|
|
8
8
|
/**
|
|
9
9
|
* The speech track's loudness, measured and normalized in two passes of ffmpeg's `loudnorm` (EBU R128): the first
|
|
10
10
|
* pass measures, the second applies one linear gain from that measurement, so a quiet phone clip is not left quiet
|
|
11
|
-
* and a loud one does not clip. It runs before the music is mixed under the
|
|
11
|
+
* and a loud one does not clip. It runs on the speech track before the music is mixed under it, and on the finished
|
|
12
|
+
* film's mix (`FootageFilm.mixLoudness`), so the published file plays at the target whatever the music adds.
|
|
12
13
|
*/
|
|
13
14
|
|
|
14
15
|
export interface LoudnessTarget {
|
|
@@ -48,9 +49,9 @@ function getLoudnormFilter(target: LoudnessTarget): string {
|
|
|
48
49
|
return `loudnorm=I=${target.target}:TP=${target.truePeak}:LRA=${LOUDNESS_RANGE}`;
|
|
49
50
|
}
|
|
50
51
|
|
|
51
|
-
/** The first pass: decodes the
|
|
52
|
+
/** The first pass: decodes the sound only (a film's picture is skipped), prints the measurement on stderr, writes nothing. */
|
|
52
53
|
export function buildLoudnessMeasureArgs(input: string, target: LoudnessTarget): string[] {
|
|
53
|
-
return ["-hide_banner", "-nostats", "-i", input, "-af", `${getLoudnormFilter(target)}:print_format=json`, "-f", "null", "-"];
|
|
54
|
+
return ["-hide_banner", "-nostats", "-i", input, "-vn", "-af", `${getLoudnormFilter(target)}:print_format=json`, "-f", "null", "-"];
|
|
54
55
|
}
|
|
55
56
|
|
|
56
57
|
/** The measurement from the first pass's stderr (its last JSON block); null when there is none or it is incomplete. */
|
|
@@ -70,8 +71,12 @@ export function readLoudnessMeasurement(stderr: string): LoudnessMeasurement | n
|
|
|
70
71
|
const parsed = measurementSchema.safeParse(json);
|
|
71
72
|
if (!parsed.success) return null;
|
|
72
73
|
const value = parsed.data;
|
|
73
|
-
// `-inf`
|
|
74
|
-
const read = (text: string): number =>
|
|
74
|
+
// Silence prints `-inf` loudness and an `inf` offset; both read as infinities, which `isSilent` tells apart.
|
|
75
|
+
const read = (text: string): number => {
|
|
76
|
+
const trimmed = text.trim();
|
|
77
|
+
if (trimmed === "-inf") return -Infinity;
|
|
78
|
+
return trimmed === "inf" || trimmed === "+inf" ? Infinity : Number(trimmed);
|
|
79
|
+
};
|
|
75
80
|
const measurement = {
|
|
76
81
|
inputI: read(value.input_i),
|
|
77
82
|
inputTp: read(value.input_tp),
|
|
@@ -87,11 +92,10 @@ export function isSilent(measurement: LoudnessMeasurement): boolean {
|
|
|
87
92
|
return !Number.isFinite(measurement.inputI) || measurement.inputI < SILENCE_LUFS || !Number.isFinite(measurement.inputThresh);
|
|
88
93
|
}
|
|
89
94
|
|
|
90
|
-
/** The second pass: one linear gain from the measurement, resampled back to the film's rate (`loudnorm` runs at 192 kHz). */
|
|
91
|
-
export function
|
|
92
|
-
const { measurement } = options;
|
|
95
|
+
/** The second pass's filter: one linear gain from the measurement, resampled back to the film's rate (`loudnorm` runs at 192 kHz). */
|
|
96
|
+
export function getLoudnessFilter(measurement: LoudnessMeasurement, target: LoudnessTarget): string {
|
|
93
97
|
const filter = [
|
|
94
|
-
getLoudnormFilter(
|
|
98
|
+
getLoudnormFilter(target),
|
|
95
99
|
`measured_I=${measurement.inputI}`,
|
|
96
100
|
`measured_TP=${measurement.inputTp}`,
|
|
97
101
|
`measured_LRA=${measurement.inputLra}`,
|
|
@@ -99,6 +103,11 @@ export function buildLoudnessArgs(options: { input: string; measurement: Loudnes
|
|
|
99
103
|
`offset=${measurement.targetOffset}`,
|
|
100
104
|
"linear=true",
|
|
101
105
|
].join(":");
|
|
106
|
+
return `${filter},aresample=${FOOTAGE_SAMPLE_RATE}`;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** The speech track's second pass, written as an mp3 at the film's rate. */
|
|
110
|
+
export function buildLoudnessArgs(options: { input: string; measurement: LoudnessMeasurement; target: LoudnessTarget; output: string }): string[] {
|
|
102
111
|
return [
|
|
103
112
|
"-v",
|
|
104
113
|
"error",
|
|
@@ -106,7 +115,7 @@ export function buildLoudnessArgs(options: { input: string; measurement: Loudnes
|
|
|
106
115
|
"-i",
|
|
107
116
|
options.input,
|
|
108
117
|
"-af",
|
|
109
|
-
|
|
118
|
+
getLoudnessFilter(options.measurement, options.target),
|
|
110
119
|
"-ar",
|
|
111
120
|
String(FOOTAGE_SAMPLE_RATE),
|
|
112
121
|
"-c:a",
|
|
@@ -117,6 +126,22 @@ export function buildLoudnessArgs(options: { input: string; measurement: Loudnes
|
|
|
117
126
|
];
|
|
118
127
|
}
|
|
119
128
|
|
|
129
|
+
export type LoudnessMeasureOutcome = { kind: "measured"; measurement: LoudnessMeasurement } | { kind: "silent" };
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* The first pass over `input`: its measurement, or silent when no gain can bring it to the target. Throws naming
|
|
133
|
+
* `label` (what is measured) when ffmpeg fails, rejects with the signal's reason on abort.
|
|
134
|
+
*/
|
|
135
|
+
export async function measureLoudness(input: string, target: LoudnessTarget, options: { label: string; signal?: AbortSignal }): Promise<LoudnessMeasureOutcome> {
|
|
136
|
+
const step = `Measuring ${options.label}'s loudness`;
|
|
137
|
+
const measured = await runProcess({ command: "ffmpeg", args: buildLoudnessMeasureArgs(input, target), ...(options.signal === undefined ? {} : { signal: options.signal }) });
|
|
138
|
+
if (measured.kind === "not-started") throw new Error(`${step}: ffmpeg did not start (${measured.error}).`);
|
|
139
|
+
if (measured.code !== 0) throw new Error(`${step}: ffmpeg ended with code ${String(measured.code)}: ${measured.stderr.trim().split("\n").slice(-3).join(" | ")}.`);
|
|
140
|
+
const measurement = readLoudnessMeasurement(measured.stderr);
|
|
141
|
+
if (measurement === null) throw new Error(`${step}: ffmpeg printed no loudnorm measurement for ${input}.`);
|
|
142
|
+
return isSilent(measurement) ? { kind: "silent" } : { kind: "measured", measurement };
|
|
143
|
+
}
|
|
144
|
+
|
|
120
145
|
export type LoudnessOutcome = { kind: "normalized"; measured: number } | { kind: "silent" };
|
|
121
146
|
|
|
122
147
|
/**
|
|
@@ -125,15 +150,12 @@ export type LoudnessOutcome = { kind: "normalized"; measured: number } | { kind:
|
|
|
125
150
|
*/
|
|
126
151
|
export async function normalizeLoudness(input: string, output: string, target: LoudnessTarget, options: { signal?: AbortSignal; onSeconds?: (seconds: number) => void } = {}): Promise<LoudnessOutcome> {
|
|
127
152
|
const signal = options.signal === undefined ? {} : { signal: options.signal };
|
|
128
|
-
const measured = await
|
|
129
|
-
if (measured.kind === "
|
|
130
|
-
if (measured.code !== 0) throw new Error(`Measuring the speech track's loudness: ffmpeg ended with code ${String(measured.code)}: ${measured.stderr.trim().split("\n").slice(-3).join(" | ")}.`);
|
|
131
|
-
const measurement = readLoudnessMeasurement(measured.stderr);
|
|
132
|
-
if (measurement === null) throw new Error(`Measuring the speech track's loudness: ffmpeg printed no loudnorm measurement for ${input}.`);
|
|
133
|
-
if (isSilent(measurement)) {
|
|
153
|
+
const measured = await measureLoudness(input, target, { label: "the speech track", ...signal });
|
|
154
|
+
if (measured.kind === "silent") {
|
|
134
155
|
copyFileSync(input, output);
|
|
135
156
|
return { kind: "silent" };
|
|
136
157
|
}
|
|
158
|
+
const { measurement } = measured;
|
|
137
159
|
await runFfmpeg("Normalizing the speech track's loudness", buildLoudnessArgs({ input, measurement, target, output }), {
|
|
138
160
|
...signal,
|
|
139
161
|
output,
|
package/src/footage/music.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { TimedWord } from "../voice/voiceover.js";
|
|
2
|
+
import type { FootageMusic } from "./film.js";
|
|
2
3
|
|
|
3
4
|
/**
|
|
4
5
|
* Music under the voice: loud between sentences, ducked while someone speaks. The envelope is a list of keyframes
|
|
@@ -22,6 +23,12 @@ export interface DuckingOptions {
|
|
|
22
23
|
gap?: number;
|
|
23
24
|
}
|
|
24
25
|
|
|
26
|
+
/** A film's music settings as ducking options: its volumes, and its attack and release when it sets them. */
|
|
27
|
+
export function getDuckingOptions(music: Pick<FootageMusic, "volume" | "duckedVolume" | "attack" | "release">): DuckingOptions {
|
|
28
|
+
const { volume, duckedVolume, attack, release } = music;
|
|
29
|
+
return { volume, duckedVolume, ...(attack === undefined ? {} : { attack }), ...(release === undefined ? {} : { release }) };
|
|
30
|
+
}
|
|
31
|
+
|
|
25
32
|
const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
|
|
26
33
|
|
|
27
34
|
/** Stretches of speech: the words joined when less than `gap` seconds apart. */
|
package/src/footage/normalize.ts
CHANGED
|
@@ -47,21 +47,47 @@ export function toneMapFilter(hdr: HdrKind | null, transfer: string | null): str
|
|
|
47
47
|
];
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
-
/**
|
|
50
|
+
/**
|
|
51
|
+
* The video filter graph from input 0's picture to `[v]`. Every stage runs on as little as it can: frames are
|
|
52
|
+
* dropped to 30 fps first, then the picture is scaled to the frame (crop: covered and cropped to it; blur: fitted
|
|
53
|
+
* inside it), and only then tone-mapped, so the float stages of an HDR clip work on frame-sized pictures instead of
|
|
54
|
+
* the source's (a 4K phone clip has 4x the pixels of a 1080x1920 frame). The blur's background is made from the
|
|
55
|
+
* fitted picture.
|
|
56
|
+
*/
|
|
51
57
|
export function buildVideoFilter(options: { fit: "crop" | "blur"; frame: Frame; hdr: HdrKind | null; transfer: string | null }): string {
|
|
52
58
|
const { width, height } = options.frame;
|
|
53
|
-
const
|
|
54
|
-
const cover = `scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}`;
|
|
59
|
+
const toneMap = toneMapFilter(options.hdr, options.transfer);
|
|
55
60
|
const tail = "setsar=1,format=yuv420p";
|
|
56
|
-
if (options.fit === "crop")
|
|
61
|
+
if (options.fit === "crop") {
|
|
62
|
+
const cover = `scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}`;
|
|
63
|
+
return `[0:v]${[`fps=${FOOTAGE_FPS}`, cover, ...toneMap, tail].join(",")}[v]`;
|
|
64
|
+
}
|
|
65
|
+
// Even sizes: zscale refuses odd ones in 4:2:0.
|
|
66
|
+
const fitted = `scale=${width}:${height}:force_original_aspect_ratio=decrease:force_divisible_by=2`;
|
|
67
|
+
const backWidth = Math.round(width / 4);
|
|
68
|
+
const backHeight = Math.round(height / 4);
|
|
57
69
|
return [
|
|
58
|
-
`[0:v]${
|
|
59
|
-
`[bg]scale=${
|
|
60
|
-
`[fg]
|
|
61
|
-
`[back][front]overlay=(W-w)/2:(H-h)/2,${tail}[v]`,
|
|
70
|
+
`[0:v]${[`fps=${FOOTAGE_FPS}`, fitted, ...toneMap, "split=2[bg][fg]"].join(",")}`,
|
|
71
|
+
`[bg]scale=${backWidth}:${backHeight}:force_original_aspect_ratio=increase,crop=${backWidth}:${backHeight},boxblur=20:2,scale=${width}:${height},eq=brightness=-0.06[back]`,
|
|
72
|
+
`[back][fg]overlay=(W-w)/2:(H-h)/2,${tail}[v]`,
|
|
62
73
|
].join(";");
|
|
63
74
|
}
|
|
64
75
|
|
|
76
|
+
/** x264's presets, fastest first. */
|
|
77
|
+
export const X264_PRESETS = ["ultrafast", "superfast", "veryfast", "faster", "fast", "medium", "slow", "slower", "veryslow"] as const;
|
|
78
|
+
export type X264Preset = (typeof X264_PRESETS)[number];
|
|
79
|
+
const DEFAULT_NORMALIZE_PRESET: X264Preset = "veryfast";
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* How the intermediate clips are encoded. A small worker can trade file size for speed: `preset` (default
|
|
83
|
+
* `veryfast`; `superfast` or `ultrafast` encode faster into bigger files), `threads` (ffmpeg's encoder and filter
|
|
84
|
+
* threads; by default ffmpeg's own choice, which follows the host's cores, not a container's CPU quota).
|
|
85
|
+
*/
|
|
86
|
+
export interface NormalizeEncoder {
|
|
87
|
+
preset?: X264Preset;
|
|
88
|
+
threads?: number;
|
|
89
|
+
}
|
|
90
|
+
|
|
65
91
|
export interface NormalizeInput {
|
|
66
92
|
clip: FootageClip;
|
|
67
93
|
/** The local file (a URL's downloaded copy). */
|
|
@@ -69,19 +95,33 @@ export interface NormalizeInput {
|
|
|
69
95
|
probe: FootageProbe;
|
|
70
96
|
frame: Frame;
|
|
71
97
|
output: string;
|
|
98
|
+
encoder?: NormalizeEncoder;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The thread arguments: global filter threads (before the inputs) and encoder threads (before the output; the same
|
|
103
|
+
* flag before an input sets that input's decoder threads). Throws, naming `option`, when `threads` is not a positive
|
|
104
|
+
* whole number.
|
|
105
|
+
*/
|
|
106
|
+
export function getThreadArgs(threads: number | undefined, option: string): { global: string[]; output: string[] } {
|
|
107
|
+
if (threads === undefined) return { global: [], output: [] };
|
|
108
|
+
if (!Number.isInteger(threads) || threads < 1) throw new Error(`${option} must be a positive whole number, got ${threads}.`);
|
|
109
|
+
return { global: ["-filter_complex_threads", String(threads)], output: ["-threads", String(threads)] };
|
|
72
110
|
}
|
|
73
111
|
|
|
74
|
-
/** ffmpeg's arguments for one clip; pure, so the graph is tested without ffmpeg. */
|
|
112
|
+
/** ffmpeg's arguments for one clip; pure, so the graph is tested without ffmpeg. Throws on an invalid `encoder.threads`. */
|
|
75
113
|
export function buildNormalizeArgs(input: NormalizeInput): string[] {
|
|
76
114
|
const { clip, file, probe, frame, output } = input;
|
|
77
115
|
const duration = getTrimmedDuration(clip, probe);
|
|
78
116
|
const fit = resolveFit(clip.fit, probe, frame);
|
|
79
117
|
const seek = clip.trim.start > 0 ? ["-ss", String(clip.trim.start)] : [];
|
|
118
|
+
const threads = getThreadArgs(input.encoder?.threads, "normalize.threads");
|
|
80
119
|
const silence = probe.hasAudio ? [] : ["-f", "lavfi", "-i", `anullsrc=r=${FOOTAGE_SAMPLE_RATE}:cl=stereo`];
|
|
81
120
|
return [
|
|
82
121
|
"-v",
|
|
83
122
|
"error",
|
|
84
123
|
"-y",
|
|
124
|
+
...threads.global,
|
|
85
125
|
...seek,
|
|
86
126
|
"-i",
|
|
87
127
|
file,
|
|
@@ -97,7 +137,7 @@ export function buildNormalizeArgs(input: NormalizeInput): string[] {
|
|
|
97
137
|
"-c:v",
|
|
98
138
|
"libx264",
|
|
99
139
|
"-preset",
|
|
100
|
-
|
|
140
|
+
input.encoder?.preset ?? DEFAULT_NORMALIZE_PRESET,
|
|
101
141
|
"-crf",
|
|
102
142
|
"18",
|
|
103
143
|
"-g",
|
|
@@ -118,6 +158,7 @@ export function buildNormalizeArgs(input: NormalizeInput): string[] {
|
|
|
118
158
|
"2",
|
|
119
159
|
"-movflags",
|
|
120
160
|
"+faststart",
|
|
161
|
+
...threads.output,
|
|
121
162
|
output,
|
|
122
163
|
];
|
|
123
164
|
}
|
package/src/footage/paid.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { err, ok, type TtsResult } from "../voice/provider.js";
|
|
|
6
6
|
import { footageCacheKey, getFootageCachePath, hashFile } from "./cache.js";
|
|
7
7
|
import { END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
|
|
8
8
|
import { readEdl, type Edl } from "./edl.js";
|
|
9
|
+
import { getFilmKeyTerms } from "./keyterms.js";
|
|
9
10
|
import type { FootageClip, FootageFilm } from "./film.js";
|
|
10
11
|
import { getTrimmedDuration, probeFootage } from "./probe.js";
|
|
11
12
|
import type { FootageProviders, FootageRates, PaidStepKind } from "./providers.js";
|
|
@@ -66,7 +67,10 @@ export function getIsolatedCache(film: FootageFilm, clipKey: string, options: Pi
|
|
|
66
67
|
|
|
67
68
|
export function getTranscriptCache(film: FootageFilm, clipKey: string, options: Pick<FootageCacheOptions, "cacheDir" | "providers">): CacheEntry {
|
|
68
69
|
const { stt } = options.providers;
|
|
69
|
-
const
|
|
70
|
+
const keyTerms = getFilmKeyTerms(film);
|
|
71
|
+
// Key terms join the key only when there are any, so a transcript bought without them keeps its key.
|
|
72
|
+
const withKeyTerms = keyTerms.ok && keyTerms.terms.length > 0 ? { keyTerms: keyTerms.terms } : {};
|
|
73
|
+
const key = footageCacheKey("transcript", { clip: clipKey, isolated: film.isolateVoice, provider: stt.id, model: stt.model, language: film.language, ...withKeyTerms });
|
|
70
74
|
return { key, path: getFootageCachePath(options.cacheDir, film.id, "transcript", key) };
|
|
71
75
|
}
|
|
72
76
|
|
|
@@ -175,10 +179,15 @@ const round = (value: number, digits: number): number => {
|
|
|
175
179
|
* What a footage film will cost in the caller's credits, before anything runs: every clip probed (a URL clip not yet
|
|
176
180
|
* downloaded is probed at its URL), nothing normalized, no provider called. Talking clips need speech to text (and
|
|
177
181
|
* isolation when the film cleans the voice); generated music lasts the cut's length when the cut is known, else the
|
|
178
|
-
* clips' length plus the end card. A cached step costs 0.
|
|
182
|
+
* clips' length plus the end card. A cached step costs 0. With key terms, transcription costs `rates.transcribe` plus
|
|
183
|
+
* the keyterm surcharge `rates.keyTerms`; invalid key terms fail the estimate.
|
|
179
184
|
*/
|
|
180
185
|
export async function estimateFootageFilm(film: FootageFilm, options: EstimateFootageOptions): Promise<TtsResult<FootageEstimate>> {
|
|
181
186
|
if (film.clips.length === 0) return err(`footage film ${film.id} has no clips.`);
|
|
187
|
+
const keyTerms = getFilmKeyTerms(film);
|
|
188
|
+
if (!keyTerms.ok) return err(`footage film ${film.id}: ${keyTerms.errors.join(" ")}`);
|
|
189
|
+
const surcharge = keyTerms.terms.length === 0 ? 0 : (options.rates.keyTerms ?? 0);
|
|
190
|
+
const rates: Record<PaidStepKind, number> = { ...options.rates, transcribe: options.rates.transcribe + surcharge };
|
|
182
191
|
const files: Record<string, string> = {};
|
|
183
192
|
for (const clip of film.clips) {
|
|
184
193
|
const local = findLocalSource(clip, options.buildDir);
|
|
@@ -189,7 +198,7 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
|
|
|
189
198
|
if (!probed.ok) return err(probed.errors.join("\n"));
|
|
190
199
|
const items: FootageEstimateItem[] = [];
|
|
191
200
|
const add = (step: PaidStepKind, item: string, provider: string, units: number, cached: boolean): void => {
|
|
192
|
-
items.push({ step, item, provider, units: round(units, 3), unit: "seconds", credits: cached ? 0 : round(units *
|
|
201
|
+
items.push({ step, item, provider, units: round(units, 3), unit: "seconds", credits: cached ? 0 : round(units * rates[step], 2), cached });
|
|
193
202
|
};
|
|
194
203
|
let clipSeconds = 0;
|
|
195
204
|
for (const { clip, probe } of probed.clips) {
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { getSegmentScales } from "./compose.js";
|
|
2
|
+
import { getEdlDuration, getSegmentStarts, type Edl } from "./edl.js";
|
|
3
|
+
import { getFootageEdit, type FootageFilm } from "./film.js";
|
|
4
|
+
import { FOOTAGE_FPS, getThreadArgs, type NormalizeEncoder } from "./normalize.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The cut's picture in film time, which the render's composition plays as one `<video>`: one short ffmpeg per segment
|
|
8
|
+
* (run one after the other, so one decoder at a time) and a stream-copy join. hyperframes then extracts the whole
|
|
9
|
+
* film's frames with one ffmpeg, and its Chrome holds one video instead of one per segment (issue #1030: the
|
|
10
|
+
* extraction peak falls from about 1 GB to 0.2 GB, the capture's from 1.7 GB to 1.1 GB on a 45 s 1080x1920 film).
|
|
11
|
+
*
|
|
12
|
+
* Every piece starts on a film frame and runs to the next segment's frame, so rounding never adds up along the cut.
|
|
13
|
+
* A cross-fade is drawn into the incoming piece: the outgoing segment's tail, held at the zoom it ends on, dissolves
|
|
14
|
+
* into the incoming clip (`xfade`), as the per-segment composition plays it. Zoom cuts and Ken Burns stay tweens on
|
|
15
|
+
* the composition's video.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
/** The point a zoom keeps still, as the composition's `transform-origin: 50% 42%`. */
|
|
19
|
+
const ZOOM_ORIGIN = { x: 0.5, y: 0.42 } as const;
|
|
20
|
+
/** How far a segment marked `kenburns` zooms when the film's `kenBurnsPercent` is 0, as in the composition. */
|
|
21
|
+
const HAND_KEN_BURNS_PERCENT = 4;
|
|
22
|
+
const PICTURE_CRF = 18;
|
|
23
|
+
const DEFAULT_PICTURE_PRESET = "veryfast";
|
|
24
|
+
|
|
25
|
+
const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
|
|
26
|
+
|
|
27
|
+
/** The outgoing segment's tail under a cross-fade. */
|
|
28
|
+
export interface PictureFadeTail {
|
|
29
|
+
clip: string;
|
|
30
|
+
/** Seconds in the clip: where the outgoing segment ends. */
|
|
31
|
+
start: number;
|
|
32
|
+
frames: number;
|
|
33
|
+
/** The zoom the outgoing segment ends on, held through the fade. */
|
|
34
|
+
scale: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** One segment's piece of the picture. */
|
|
38
|
+
export interface PicturePiece {
|
|
39
|
+
clip: string;
|
|
40
|
+
/** Seconds in the clip where the piece starts. */
|
|
41
|
+
start: number;
|
|
42
|
+
/** The piece's first film frame. */
|
|
43
|
+
frame: number;
|
|
44
|
+
frames: number;
|
|
45
|
+
/** For a `fade` cut, the outgoing tail the piece dissolves from; null for any other cut. */
|
|
46
|
+
fade: PictureFadeTail | null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The film frame a film second falls on. */
|
|
50
|
+
function toFrame(seconds: number): number {
|
|
51
|
+
return Math.round(seconds * FOOTAGE_FPS);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The picture's pieces, one per segment, covering the cut's frames without a gap. Throws on an EDL without segments. */
|
|
55
|
+
export function planFootagePicture(film: Pick<FootageFilm, "edit">, edl: Edl): PicturePiece[] {
|
|
56
|
+
if (edl.segments.length === 0) throw new Error("The EDL has no segments.");
|
|
57
|
+
const edit = getFootageEdit(film);
|
|
58
|
+
const kenBurnsScale = 1 + (edit.kenBurnsPercent > 0 ? edit.kenBurnsPercent : HAND_KEN_BURNS_PERCENT) / 100;
|
|
59
|
+
const scales = getSegmentScales(edl, edit.zoomPercent);
|
|
60
|
+
const starts = getSegmentStarts(edl);
|
|
61
|
+
const lastFrame = toFrame(getEdlDuration(edl));
|
|
62
|
+
return edl.segments.map((segment, index) => {
|
|
63
|
+
const frame = toFrame(starts[index] as number);
|
|
64
|
+
const nextStart = starts[index + 1];
|
|
65
|
+
const frames = Math.max(1, (nextStart === undefined ? lastFrame : toFrame(nextStart)) - frame);
|
|
66
|
+
const previous = edl.segments[index - 1];
|
|
67
|
+
const fadeFrames = segment.transition === "fade" ? Math.min(frames, edit.fadeFrames) : 0;
|
|
68
|
+
let fade: PictureFadeTail | null = null;
|
|
69
|
+
if (previous !== undefined && fadeFrames > 0) {
|
|
70
|
+
const previousScale = scales[index - 1] as number;
|
|
71
|
+
// The outgoing segment's Ken Burns tween ends at its end; its tail holds the zoom it reached.
|
|
72
|
+
const scale = previous.motion === "kenburns" ? Math.round(previousScale * kenBurnsScale * 10000) / 10000 : previousScale;
|
|
73
|
+
fade = { clip: previous.clip, start: r3(previous.end), frames: fadeFrames, scale };
|
|
74
|
+
}
|
|
75
|
+
return { clip: segment.clip, start: r3(segment.start), frame, frames, fade };
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** The number of frames the picture has: the cut's length in film frames. */
|
|
80
|
+
export function getPictureFrames(pieces: readonly PicturePiece[]): number {
|
|
81
|
+
return pieces.reduce((sum, piece) => sum + piece.frames, 0);
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Filters that bring a stream to the film's rate and pixel format and exactly `frames` long (a short clip holds its last frame). */
|
|
85
|
+
function fitStream(frames: number): string[] {
|
|
86
|
+
const seconds = r3(frames / FOOTAGE_FPS);
|
|
87
|
+
return [
|
|
88
|
+
"setpts=PTS-STARTPTS",
|
|
89
|
+
`fps=${FOOTAGE_FPS}`,
|
|
90
|
+
"format=yuv420p",
|
|
91
|
+
"setsar=1",
|
|
92
|
+
`tpad=stop_mode=clone:stop_duration=${seconds}`,
|
|
93
|
+
`trim=end_frame=${frames}`,
|
|
94
|
+
"setpts=PTS-STARTPTS",
|
|
95
|
+
`settb=1/${FOOTAGE_FPS}`,
|
|
96
|
+
];
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** A crop to `scale` about the zoom origin and back to the clip's size; nothing at scale 1. */
|
|
100
|
+
function zoomFilters(scale: number): string[] {
|
|
101
|
+
if (scale === 1) return [];
|
|
102
|
+
return [`crop=iw/${scale}:ih/${scale}:(iw-iw/${scale})*${ZOOM_ORIGIN.x}:(ih-ih/${scale})*${ZOOM_ORIGIN.y}`, `scale=iw*${scale}:ih*${scale}`];
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export interface PicturePieceArgsInput {
|
|
106
|
+
piece: PicturePiece;
|
|
107
|
+
/** The normalized clips by id. */
|
|
108
|
+
clips: Record<string, string>;
|
|
109
|
+
output: string;
|
|
110
|
+
/** The intermediate clips' encoder (`RenderFootageOptions.normalize`): its preset and threads encode the piece too. */
|
|
111
|
+
encoder?: NormalizeEncoder;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** One piece's ffmpeg arguments: its clip (and a fade's outgoing tail), exactly `frames` frames, H.264 without sound. */
|
|
115
|
+
export function buildPicturePieceArgs(input: PicturePieceArgsInput): string[] {
|
|
116
|
+
const { piece, clips } = input;
|
|
117
|
+
const getClip = (id: string): string => {
|
|
118
|
+
const file = clips[id];
|
|
119
|
+
if (file === undefined) throw new Error(`No file for clip "${id}" of the EDL.`);
|
|
120
|
+
return file;
|
|
121
|
+
};
|
|
122
|
+
const threads = getThreadArgs(input.encoder?.threads, "normalize.threads");
|
|
123
|
+
const seconds = (frames: number): string => String(r3(frames / FOOTAGE_FPS + 1 / FOOTAGE_FPS));
|
|
124
|
+
const inputs = [...threads.output, "-ss", String(piece.start), "-t", seconds(piece.frames), "-i", getClip(piece.clip)];
|
|
125
|
+
let graph = `[0:v]${fitStream(piece.frames).join(",")}[picture]`;
|
|
126
|
+
if (piece.fade !== null) {
|
|
127
|
+
const { fade } = piece;
|
|
128
|
+
inputs.push(...threads.output, "-ss", String(fade.start), "-t", seconds(fade.frames), "-i", getClip(fade.clip));
|
|
129
|
+
const tail = [...zoomFilters(fade.scale), ...fitStream(fade.frames)];
|
|
130
|
+
graph = [
|
|
131
|
+
`[0:v]${fitStream(piece.frames).join(",")}[incoming]`,
|
|
132
|
+
`[1:v]${tail.join(",")}[outgoing]`,
|
|
133
|
+
// xfade's output is the outgoing tail plus the incoming piece minus the fade: the piece's own length.
|
|
134
|
+
`[outgoing][incoming]xfade=transition=fade:duration=${r3(fade.frames / FOOTAGE_FPS)}:offset=0,trim=end_frame=${piece.frames}[picture]`,
|
|
135
|
+
].join(";");
|
|
136
|
+
}
|
|
137
|
+
return [
|
|
138
|
+
"-v",
|
|
139
|
+
"error",
|
|
140
|
+
"-y",
|
|
141
|
+
...threads.global,
|
|
142
|
+
...inputs,
|
|
143
|
+
"-filter_complex",
|
|
144
|
+
graph,
|
|
145
|
+
"-map",
|
|
146
|
+
"[picture]",
|
|
147
|
+
"-an",
|
|
148
|
+
"-frames:v",
|
|
149
|
+
String(piece.frames),
|
|
150
|
+
"-c:v",
|
|
151
|
+
"libx264",
|
|
152
|
+
"-preset",
|
|
153
|
+
input.encoder?.preset ?? DEFAULT_PICTURE_PRESET,
|
|
154
|
+
"-crf",
|
|
155
|
+
String(PICTURE_CRF),
|
|
156
|
+
"-g",
|
|
157
|
+
String(FOOTAGE_FPS / 2),
|
|
158
|
+
"-pix_fmt",
|
|
159
|
+
"yuv420p",
|
|
160
|
+
"-color_primaries",
|
|
161
|
+
"bt709",
|
|
162
|
+
"-color_trc",
|
|
163
|
+
"bt709",
|
|
164
|
+
"-colorspace",
|
|
165
|
+
"bt709",
|
|
166
|
+
...threads.output,
|
|
167
|
+
input.output,
|
|
168
|
+
];
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** The concat demuxer's list: one `file` line per piece, quotes escaped as the demuxer reads them. */
|
|
172
|
+
export function buildPictureList(files: readonly string[]): string {
|
|
173
|
+
return `${files.map((file) => `file '${file.replace(/'/g, "'\\''")}'`).join("\n")}\n`;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Joins the pieces by stream copy (they share one encoder's settings) into the picture, fast start. */
|
|
177
|
+
export function buildPictureConcatArgs(list: string, output: string): string[] {
|
|
178
|
+
return ["-v", "error", "-y", "-f", "concat", "-safe", "0", "-i", list, "-c", "copy", "-movflags", "+faststart", output];
|
|
179
|
+
}
|