@softure-ai/marketing-kit 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +135 -14
- package/dist/cli/footage.d.ts.map +1 -1
- package/dist/cli/footage.js +15 -6
- package/dist/cli/footage.js.map +1 -1
- package/dist/config/actions-schema.d.ts +14 -14
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +5 -0
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +92 -69
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +40 -4
- package/dist/config/schema.js.map +1 -1
- package/dist/config/shot-steps.d.ts +16 -16
- package/dist/footage/audio.d.ts +52 -0
- package/dist/footage/audio.d.ts.map +1 -0
- package/dist/footage/audio.js +100 -0
- package/dist/footage/audio.js.map +1 -0
- package/dist/footage/captions.d.ts +57 -0
- package/dist/footage/captions.d.ts.map +1 -0
- package/dist/footage/captions.js +112 -0
- package/dist/footage/captions.js.map +1 -0
- package/dist/footage/compose.d.ts +5 -5
- package/dist/footage/compose.d.ts.map +1 -1
- package/dist/footage/compose.js +31 -22
- package/dist/footage/compose.js.map +1 -1
- package/dist/footage/edl.d.ts +54 -16
- package/dist/footage/edl.d.ts.map +1 -1
- package/dist/footage/edl.js +204 -52
- package/dist/footage/edl.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts +4 -1
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +18 -6
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/fake.d.ts +2 -0
- package/dist/footage/fake.d.ts.map +1 -1
- package/dist/footage/fake.js +2 -1
- package/dist/footage/fake.js.map +1 -1
- package/dist/footage/film.d.ts +35 -1
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js +9 -1
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/frames.d.ts +38 -0
- package/dist/footage/frames.d.ts.map +1 -0
- package/dist/footage/frames.js +40 -0
- package/dist/footage/frames.js.map +1 -0
- package/dist/footage/loudness.d.ts +49 -0
- package/dist/footage/loudness.d.ts.map +1 -0
- package/dist/footage/loudness.js +114 -0
- package/dist/footage/loudness.js.map +1 -0
- package/dist/footage/normalize.d.ts +6 -2
- package/dist/footage/normalize.d.ts.map +1 -1
- package/dist/footage/normalize.js +7 -9
- package/dist/footage/normalize.js.map +1 -1
- package/dist/footage/paid.d.ts +70 -0
- package/dist/footage/paid.d.ts.map +1 -0
- package/dist/footage/paid.js +172 -0
- package/dist/footage/paid.js.map +1 -0
- package/dist/footage/pipeline.d.ts +34 -10
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +294 -85
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/probe.d.ts +18 -4
- package/dist/footage/probe.d.ts.map +1 -1
- package/dist/footage/probe.js +18 -13
- package/dist/footage/probe.js.map +1 -1
- package/dist/footage/process.d.ts +45 -0
- package/dist/footage/process.d.ts.map +1 -0
- package/dist/footage/process.js +109 -0
- package/dist/footage/process.js.map +1 -0
- package/dist/footage/providers.d.ts +49 -5
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js +4 -0
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/publish.d.ts +62 -0
- package/dist/footage/publish.d.ts.map +1 -0
- package/dist/footage/publish.js +164 -0
- package/dist/footage/publish.js.map +1 -0
- package/dist/footage/render.d.ts +5 -4
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +29 -17
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/sentences.d.ts +44 -0
- package/dist/footage/sentences.d.ts.map +1 -0
- package/dist/footage/sentences.js +102 -0
- package/dist/footage/sentences.js.map +1 -0
- package/dist/footage/tools.d.ts +33 -0
- package/dist/footage/tools.d.ts.map +1 -0
- package/dist/footage/tools.js +108 -0
- package/dist/footage/tools.js.map +1 -0
- package/dist/footage/words.d.ts +6 -0
- package/dist/footage/words.d.ts.map +1 -0
- package/dist/footage/words.js +14 -0
- package/dist/footage/words.js.map +1 -0
- package/dist/index.d.ts +15 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -6
- package/dist/index.js.map +1 -1
- package/docker/footage-worker.Dockerfile +29 -0
- package/package.json +2 -1
- package/schema/edl.schema.json +115 -0
- package/schema/marketing.schema.json +112 -5
- package/src/cli/footage.ts +13 -5
- package/src/config/config.ts +5 -0
- package/src/config/schema.ts +40 -4
- package/src/footage/audio.ts +123 -0
- package/src/footage/captions.ts +146 -0
- package/src/footage/compose.ts +32 -24
- package/src/footage/edl.ts +225 -56
- package/src/footage/elevenlabs.ts +21 -8
- package/src/footage/fake.ts +4 -1
- package/src/footage/film.ts +43 -2
- package/src/footage/frames.ts +65 -0
- package/src/footage/loudness.ts +143 -0
- package/src/footage/normalize.ts +7 -9
- package/src/footage/paid.ts +213 -0
- package/src/footage/pipeline.ts +344 -89
- package/src/footage/probe.ts +20 -12
- package/src/footage/process.ts +135 -0
- package/src/footage/providers.ts +56 -6
- package/src/footage/publish.ts +180 -0
- package/src/footage/render.ts +29 -19
- package/src/footage/sentences.ts +137 -0
- package/src/footage/tools.ts +119 -0
- package/src/footage/words.ts +15 -0
- package/src/index.ts +84 -0
package/src/footage/probe.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import { spawnSync } from "node:child_process";
|
|
2
|
-
|
|
3
1
|
import { z } from "zod";
|
|
4
2
|
|
|
5
3
|
import type { FootageClip } from "./film.js";
|
|
4
|
+
import { runProcess } from "./process.js";
|
|
6
5
|
|
|
7
6
|
/**
|
|
8
7
|
* What ffprobe says about a clip, reduced to what the footage pipeline needs: its length, its picture as it is
|
|
@@ -102,18 +101,23 @@ export function readFootageProbe(json: unknown): ProbeResult {
|
|
|
102
101
|
};
|
|
103
102
|
}
|
|
104
103
|
|
|
105
|
-
/** Runs ffprobe on one file; an error names the file
|
|
106
|
-
export function
|
|
107
|
-
const result =
|
|
108
|
-
if (result.
|
|
109
|
-
if (result.
|
|
110
|
-
let json: unknown;
|
|
104
|
+
/** Runs ffprobe (`-show_streams -show_format -of json`) on one file or URL and parses its JSON; an error names the file. Rejects only on abort. */
|
|
105
|
+
export async function runFfprobe(path: string, options: { signal?: AbortSignal } = {}): Promise<{ ok: true; json: unknown } | { ok: false; error: string }> {
|
|
106
|
+
const result = await runProcess({ command: "ffprobe", args: ["-v", "error", "-show_streams", "-show_format", "-of", "json", path], ...options });
|
|
107
|
+
if (result.kind === "not-started") return { ok: false, error: `ffprobe did not start for ${path}: ${result.error}` };
|
|
108
|
+
if (result.code !== 0) return { ok: false, error: `ffprobe cannot read ${path}: ${result.stderr.trim() || `code ${String(result.code)}`}` };
|
|
111
109
|
try {
|
|
112
|
-
json
|
|
110
|
+
return { ok: true, json: JSON.parse(result.stdout) as unknown };
|
|
113
111
|
} catch (error) {
|
|
114
112
|
return { ok: false, error: `ffprobe printed no JSON for ${path}: ${String(error)}` };
|
|
115
113
|
}
|
|
116
|
-
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Runs ffprobe on one file (or URL) without blocking; an error names the file and the reason. Rejects only on abort. */
|
|
117
|
+
export async function probeFile(path: string, options: { signal?: AbortSignal } = {}): Promise<ProbeResult> {
|
|
118
|
+
const output = await runFfprobe(path, options);
|
|
119
|
+
if (!output.ok) return output;
|
|
120
|
+
const read = readFootageProbe(output.json);
|
|
117
121
|
return read.ok ? read : { ok: false, error: `${path}: ${read.error}` };
|
|
118
122
|
}
|
|
119
123
|
|
|
@@ -146,12 +150,16 @@ export interface ProbedClip {
|
|
|
146
150
|
* Probes every clip and reports every problem at once, before anything is paid for. `files` maps a clip id to its
|
|
147
151
|
* local file (a URL's downloaded copy).
|
|
148
152
|
*/
|
|
149
|
-
export function probeFootage(
|
|
153
|
+
export async function probeFootage(
|
|
154
|
+
clips: FootageClip[],
|
|
155
|
+
files: Record<string, string>,
|
|
156
|
+
options: { signal?: AbortSignal } = {},
|
|
157
|
+
): Promise<{ ok: true; clips: ProbedClip[] } | { ok: false; errors: string[] }> {
|
|
150
158
|
const probed: ProbedClip[] = [];
|
|
151
159
|
const errors: string[] = [];
|
|
152
160
|
for (const clip of clips) {
|
|
153
161
|
const file = files[clip.id] ?? clip.source;
|
|
154
|
-
const result = probeFile(file);
|
|
162
|
+
const result = await probeFile(file, options);
|
|
155
163
|
if (!result.ok) {
|
|
156
164
|
errors.push(`clip "${clip.id}": ${result.error}`);
|
|
157
165
|
continue;
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { rmSync } from "node:fs";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Child processes of the footage blocks (ffmpeg, ffprobe, hyperframes), run without blocking the event loop, so a jobs
|
|
6
|
+
* worker keeps its lease heartbeat during a long normalize or render. An abort kills the child, removes the partial
|
|
7
|
+
* files it was writing and rejects with the signal's reason, as `fetch` does.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
export interface ProcessOptions {
|
|
11
|
+
command: string;
|
|
12
|
+
args: string[];
|
|
13
|
+
signal?: AbortSignal;
|
|
14
|
+
/** Files the child writes; removed when the run is aborted. */
|
|
15
|
+
removeOnAbort?: string[];
|
|
16
|
+
/** Each line the child prints on stdout; without it stdout is collected and returned. */
|
|
17
|
+
onStdoutLine?: (line: string) => void;
|
|
18
|
+
/** Each line the child prints on stderr; without it stderr is collected and returned. */
|
|
19
|
+
onStderrLine?: (line: string) => void;
|
|
20
|
+
cwd?: string;
|
|
21
|
+
env?: NodeJS.ProcessEnv;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export type ProcessOutcome =
|
|
25
|
+
| { kind: "exited"; code: number | null; signal: NodeJS.Signals | null; stdout: string; stderr: string }
|
|
26
|
+
| { kind: "not-started"; error: string };
|
|
27
|
+
|
|
28
|
+
/** Splits a stream into lines, handing each finished line on and keeping the rest for the next chunk. */
|
|
29
|
+
function createLineSplitter(onLine: (line: string) => void): { push: (chunk: string) => void; end: () => void } {
|
|
30
|
+
let rest = "";
|
|
31
|
+
return {
|
|
32
|
+
push(chunk) {
|
|
33
|
+
const lines = (rest + chunk).split(/\r?\n|\r/);
|
|
34
|
+
rest = lines.pop() ?? "";
|
|
35
|
+
for (const line of lines) onLine(line);
|
|
36
|
+
},
|
|
37
|
+
end() {
|
|
38
|
+
if (rest.length > 0) onLine(rest);
|
|
39
|
+
rest = "";
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function removeFiles(paths: string[]): void {
|
|
45
|
+
for (const path of paths) rmSync(path, { recursive: true, force: true });
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Runs a child process to its end. Resolves with how it ended (a failure to start is a value too); rejects only when
|
|
50
|
+
* `signal` aborts, after the child is killed and `removeOnAbort` removed.
|
|
51
|
+
*/
|
|
52
|
+
export function runProcess(options: ProcessOptions): Promise<ProcessOutcome> {
|
|
53
|
+
const { signal } = options;
|
|
54
|
+
if (signal?.aborted === true) {
|
|
55
|
+
removeFiles(options.removeOnAbort ?? []);
|
|
56
|
+
return Promise.reject(signal.reason as Error);
|
|
57
|
+
}
|
|
58
|
+
return new Promise((resolve, reject) => {
|
|
59
|
+
const child = spawn(options.command, options.args, { cwd: options.cwd, env: options.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
60
|
+
let stdout = "";
|
|
61
|
+
let stderr = "";
|
|
62
|
+
const out = createLineSplitter((line) => options.onStdoutLine?.(line));
|
|
63
|
+
const err = createLineSplitter((line) => options.onStderrLine?.(line));
|
|
64
|
+
child.stdout.setEncoding("utf8").on("data", (chunk: string) => {
|
|
65
|
+
if (options.onStdoutLine === undefined) stdout += chunk;
|
|
66
|
+
else out.push(chunk);
|
|
67
|
+
});
|
|
68
|
+
child.stderr.setEncoding("utf8").on("data", (chunk: string) => {
|
|
69
|
+
if (options.onStderrLine === undefined) stderr += chunk;
|
|
70
|
+
else err.push(chunk);
|
|
71
|
+
});
|
|
72
|
+
let isAborted = false;
|
|
73
|
+
const onAbort = (): void => {
|
|
74
|
+
isAborted = true;
|
|
75
|
+
child.kill("SIGKILL");
|
|
76
|
+
};
|
|
77
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
78
|
+
let startError: string | null = null;
|
|
79
|
+
child.on("error", (error) => {
|
|
80
|
+
startError = error.message;
|
|
81
|
+
});
|
|
82
|
+
child.on("close", (code, exitSignal) => {
|
|
83
|
+
signal?.removeEventListener("abort", onAbort);
|
|
84
|
+
out.end();
|
|
85
|
+
err.end();
|
|
86
|
+
if (isAborted && signal !== undefined) {
|
|
87
|
+
removeFiles(options.removeOnAbort ?? []);
|
|
88
|
+
reject(signal.reason as Error);
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
if (startError !== null) {
|
|
92
|
+
resolve({ kind: "not-started", error: startError });
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
resolve({ kind: "exited", code, signal: exitSignal, stdout, stderr });
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Seconds ffmpeg has written so far, from a `-progress` line (`out_time_us=…`); null for any other line. */
|
|
101
|
+
export function readFfmpegProgressSeconds(line: string): number | null {
|
|
102
|
+
const match = /^out_time_us=(\d+)$/.exec(line.trim());
|
|
103
|
+
return match === null ? null : Number(match[1]) / 1_000_000;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export interface FfmpegRunOptions {
|
|
107
|
+
signal?: AbortSignal;
|
|
108
|
+
/** The file the run writes, removed on abort. */
|
|
109
|
+
output?: string;
|
|
110
|
+
/** Seconds of output written so far, as ffmpeg reports them. */
|
|
111
|
+
onSeconds?: (seconds: number) => void;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Runs ffmpeg with its progress on stdout; throws naming the step when it fails (a broken input or machine). */
|
|
115
|
+
export async function runFfmpeg(step: string, args: string[], options: FfmpegRunOptions = {}): Promise<void> {
|
|
116
|
+
const errors: string[] = [];
|
|
117
|
+
const outcome = await runProcess({
|
|
118
|
+
command: "ffmpeg",
|
|
119
|
+
args: ["-progress", "pipe:1", "-nostats", ...args],
|
|
120
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
121
|
+
removeOnAbort: options.output === undefined ? [] : [options.output],
|
|
122
|
+
onStdoutLine: (line) => {
|
|
123
|
+
const seconds = readFfmpegProgressSeconds(line);
|
|
124
|
+
if (seconds !== null) options.onSeconds?.(seconds);
|
|
125
|
+
},
|
|
126
|
+
onStderrLine: (line) => {
|
|
127
|
+
if (line.trim().length > 0) errors.push(line);
|
|
128
|
+
},
|
|
129
|
+
});
|
|
130
|
+
if (outcome.kind === "not-started") throw new Error(`${step}: ffmpeg did not start (${outcome.error}).`);
|
|
131
|
+
if (outcome.code !== 0) {
|
|
132
|
+
const detail = errors.slice(-5).join(" | ");
|
|
133
|
+
throw new Error(`${step}: ffmpeg ended with code ${String(outcome.code)} (signal ${outcome.signal ?? "-"})${detail.length > 0 ? `: ${detail}` : ""}.`);
|
|
134
|
+
}
|
|
135
|
+
}
|
package/src/footage/providers.ts
CHANGED
|
@@ -11,12 +11,22 @@ export interface SttInput {
|
|
|
11
11
|
audio: string;
|
|
12
12
|
/** ISO 639 code the speech is in. */
|
|
13
13
|
language: string;
|
|
14
|
+
/** Aborts the request. */
|
|
15
|
+
signal?: AbortSignal;
|
|
14
16
|
}
|
|
15
17
|
|
|
16
18
|
export interface SttTranscript {
|
|
17
19
|
/** Spoken words with their times in seconds from the file's start; no spacing or sound events. */
|
|
18
20
|
words: TimedWord[];
|
|
19
21
|
language: string;
|
|
22
|
+
/** What the provider reported it charged for the call, in its credits; absent when the API reports nothing. */
|
|
23
|
+
credits?: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Audio a paid call returned, with the charge the provider reported (absent when the API reports nothing). */
|
|
27
|
+
export interface PaidAudio {
|
|
28
|
+
audio: Buffer;
|
|
29
|
+
credits?: number;
|
|
20
30
|
}
|
|
21
31
|
|
|
22
32
|
/** Speech to text with a time for every word. */
|
|
@@ -27,18 +37,23 @@ export interface SttProvider {
|
|
|
27
37
|
transcribe(input: SttInput): Promise<TtsResult<SttTranscript>>;
|
|
28
38
|
}
|
|
29
39
|
|
|
30
|
-
/** Removes background noise from speech; returns the cleaned audio (mp3). */
|
|
40
|
+
/** Removes background noise from speech; returns the cleaned audio (mp3), alone or with the reported charge. */
|
|
31
41
|
export interface IsolationProvider {
|
|
32
42
|
readonly id: string;
|
|
33
|
-
isolate(input: { audio: string }): Promise<TtsResult<Buffer>>;
|
|
43
|
+
isolate(input: { audio: string; signal?: AbortSignal }): Promise<TtsResult<Buffer | PaidAudio>>;
|
|
34
44
|
}
|
|
35
45
|
|
|
36
46
|
/** Generates music (mp3): from a prompt at a length, or scored to a video. */
|
|
37
47
|
export interface MusicProvider {
|
|
38
48
|
readonly id: string;
|
|
39
49
|
readonly model: string;
|
|
40
|
-
compose(input: { prompt: string; seconds: number }): Promise<TtsResult<Buffer>>;
|
|
41
|
-
scoreVideo(input: { video: string; description: string; tags: string[] }): Promise<TtsResult<Buffer>>;
|
|
50
|
+
compose(input: { prompt: string; seconds: number; signal?: AbortSignal }): Promise<TtsResult<Buffer | PaidAudio>>;
|
|
51
|
+
scoreVideo(input: { video: string; description: string; tags: string[]; signal?: AbortSignal }): Promise<TtsResult<Buffer | PaidAudio>>;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** A provider's audio answer as `PaidAudio`, whichever of the two shapes it returned. */
|
|
55
|
+
export function toPaidAudio(value: Buffer | PaidAudio): PaidAudio {
|
|
56
|
+
return Buffer.isBuffer(value) ? { audio: value } : value;
|
|
42
57
|
}
|
|
43
58
|
|
|
44
59
|
export interface FootageProviders {
|
|
@@ -47,16 +62,18 @@ export interface FootageProviders {
|
|
|
47
62
|
music: MusicProvider;
|
|
48
63
|
}
|
|
49
64
|
|
|
65
|
+
export type PaidStepKind = "transcribe" | "isolate" | "music";
|
|
66
|
+
|
|
50
67
|
/** One paid call the film needs, as the dry run shows it. The services bill by length, so the estimate is seconds. */
|
|
51
68
|
export interface PaidStep {
|
|
52
|
-
kind:
|
|
69
|
+
kind: PaidStepKind;
|
|
53
70
|
/** What it is for: a clip id, or `film`. */
|
|
54
71
|
item: string;
|
|
55
72
|
provider: string;
|
|
56
73
|
seconds: number;
|
|
57
74
|
}
|
|
58
75
|
|
|
59
|
-
const STEP_LABELS: Record<
|
|
76
|
+
const STEP_LABELS: Record<PaidStepKind, string> = {
|
|
60
77
|
transcribe: "speech to text",
|
|
61
78
|
isolate: "voice isolation",
|
|
62
79
|
music: "music",
|
|
@@ -69,3 +86,36 @@ export function describePaidSteps(steps: PaidStep[]): string[] {
|
|
|
69
86
|
lines.push(`footage estimate: ${steps.length} paid call${steps.length === 1 ? "" : "s"}, ${total.toFixed(1)} s of audio or music billed by the providers' per-minute rates.`);
|
|
70
87
|
return lines;
|
|
71
88
|
}
|
|
89
|
+
|
|
90
|
+
/** Credits per second billed, for each paid step; the caller's prices (the kit hard-codes none). */
|
|
91
|
+
export type FootageRates = Record<PaidStepKind, number>;
|
|
92
|
+
|
|
93
|
+
/** One paid call made, as `onCharge` reports it: what for, how much was billed and where its result is cached. */
|
|
94
|
+
export interface FootageCharge {
|
|
95
|
+
step: PaidStepKind;
|
|
96
|
+
/** A clip id, or `film` for the music. */
|
|
97
|
+
item: string;
|
|
98
|
+
provider: string;
|
|
99
|
+
/** Seconds of audio or music billed. */
|
|
100
|
+
units: number;
|
|
101
|
+
/** The provider's reported charge, when its API returns one. */
|
|
102
|
+
credits?: number;
|
|
103
|
+
/** The paid cache's key of the result (the file name's first part under `<cacheDir>/<film id>/`). */
|
|
104
|
+
cacheKey: string;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** A stage of a footage run that reports progress. */
|
|
108
|
+
export type FootageStage = "normalize" | PaidStepKind | "assets" | "render" | "finish";
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Progress of a footage run. `done` and `total` are seconds of media across the whole stage (a bar per stage);
|
|
112
|
+
* `itemIndex` of `itemCount` (from 1) names the clip or asset being worked on ("normalizing clip 2 of 4").
|
|
113
|
+
*/
|
|
114
|
+
export interface FootageProgress {
|
|
115
|
+
step: FootageStage;
|
|
116
|
+
item: string;
|
|
117
|
+
itemIndex: number;
|
|
118
|
+
itemCount: number;
|
|
119
|
+
done: number;
|
|
120
|
+
total: number;
|
|
121
|
+
}
|
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import { closeSync, openSync, readSync, statSync } from "node:fs";
|
|
2
|
+
import { extname } from "node:path";
|
|
3
|
+
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
|
|
6
|
+
import { getSegmentStarts, type Edl } from "./edl.js";
|
|
7
|
+
import type { ClipRole } from "./film.js";
|
|
8
|
+
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
9
|
+
import { runProcess } from "./process.js";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The finished film as a platform or a scheduler takes it: H.264 High in yuv420p, AAC 48 kHz stereo, the `moov` box
|
|
13
|
+
* before the media (`+faststart`, so a streamed URL plays before it is whole), a cover frame, and the facts an app
|
|
14
|
+
* records (bytes, picture, real duration), all read from the written file.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
/** The audio bitrate of a re-encoded track. */
|
|
18
|
+
const AUDIO_BITRATE = "192k";
|
|
19
|
+
/** The quality of a re-encoded picture (x264's constant rate factor; 18 is visually lossless). */
|
|
20
|
+
const VIDEO_CRF = 18;
|
|
21
|
+
/** How far before the film's end the latest cover frame is taken, so a frame still exists there. */
|
|
22
|
+
const COVER_END_MARGIN = 0.1;
|
|
23
|
+
|
|
24
|
+
/** What ffprobe says about a written film, reduced to what a platform checks. */
|
|
25
|
+
export interface FilmFacts {
|
|
26
|
+
bytes: number;
|
|
27
|
+
width: number;
|
|
28
|
+
height: number;
|
|
29
|
+
durationSeconds: number;
|
|
30
|
+
video: { codec: string; profile: string | null; pixelFormat: string | null };
|
|
31
|
+
/** Null for a film without sound. */
|
|
32
|
+
audio: { codec: string; sampleRate: number; channels: number } | null;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const streamSchema = z.looseObject({
|
|
36
|
+
codec_type: z.string().optional(),
|
|
37
|
+
codec_name: z.string().optional(),
|
|
38
|
+
profile: z.string().optional(),
|
|
39
|
+
pix_fmt: z.string().optional(),
|
|
40
|
+
width: z.number().optional(),
|
|
41
|
+
height: z.number().optional(),
|
|
42
|
+
sample_rate: z.string().optional(),
|
|
43
|
+
channels: z.number().optional(),
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
const factsSchema = z.looseObject({
|
|
47
|
+
streams: z.array(streamSchema).default([]),
|
|
48
|
+
format: z.looseObject({ duration: z.string().optional(), size: z.string().optional() }).optional(),
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
/** ffprobe's JSON (`-show_streams -show_format -of json`) of a film read as its facts; an error names what is missing. */
|
|
52
|
+
export function readFilmFacts(json: unknown): { ok: true; facts: FilmFacts } | { ok: false; error: string } {
|
|
53
|
+
const parsed = factsSchema.safeParse(json);
|
|
54
|
+
if (!parsed.success) return { ok: false, error: "ffprobe printed an unexpected shape" };
|
|
55
|
+
const { streams, format } = parsed.data;
|
|
56
|
+
const video = streams.find((stream) => stream.codec_type === "video");
|
|
57
|
+
if (video?.width === undefined || video.height === undefined) return { ok: false, error: "no video stream" };
|
|
58
|
+
const durationSeconds = Number(format?.duration);
|
|
59
|
+
if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) return { ok: false, error: "no duration" };
|
|
60
|
+
const audio = streams.find((stream) => stream.codec_type === "audio");
|
|
61
|
+
return {
|
|
62
|
+
ok: true,
|
|
63
|
+
facts: {
|
|
64
|
+
bytes: Number(format?.size ?? 0),
|
|
65
|
+
width: video.width,
|
|
66
|
+
height: video.height,
|
|
67
|
+
durationSeconds,
|
|
68
|
+
video: { codec: video.codec_name ?? "unknown", profile: video.profile ?? null, pixelFormat: video.pix_fmt ?? null },
|
|
69
|
+
audio: audio === undefined ? null : { codec: audio.codec_name ?? "unknown", sampleRate: Number(audio.sample_rate ?? 0), channels: audio.channels ?? 0 },
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Runs ffprobe on a written film; an error names the file and the reason. Rejects only on abort. */
|
|
75
|
+
export async function probeFilmFacts(path: string, options: { signal?: AbortSignal } = {}): Promise<{ ok: true; facts: FilmFacts } | { ok: false; error: string }> {
|
|
76
|
+
const result = await runProcess({ command: "ffprobe", args: ["-v", "error", "-show_streams", "-show_format", "-of", "json", path], ...options });
|
|
77
|
+
if (result.kind === "not-started") return { ok: false, error: `ffprobe did not start for ${path}: ${result.error}` };
|
|
78
|
+
if (result.code !== 0) return { ok: false, error: `ffprobe cannot read ${path}: ${result.stderr.trim() || `code ${String(result.code)}`}` };
|
|
79
|
+
let json: unknown;
|
|
80
|
+
try {
|
|
81
|
+
json = JSON.parse(result.stdout);
|
|
82
|
+
} catch (error) {
|
|
83
|
+
return { ok: false, error: `ffprobe printed no JSON for ${path}: ${String(error)}` };
|
|
84
|
+
}
|
|
85
|
+
const read = readFilmFacts(json);
|
|
86
|
+
return read.ok ? read : { ok: false, error: `${path}: ${read.error}` };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** True when the picture is already what platforms take: H.264 High, yuv420p. */
|
|
90
|
+
export function isPublishableVideo(video: FilmFacts["video"]): boolean {
|
|
91
|
+
return video.codec === "h264" && video.profile === "High" && video.pixelFormat === "yuv420p";
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** True when the sound is already what platforms take: AAC at 48 kHz, stereo. */
|
|
95
|
+
export function isPublishableAudio(audio: NonNullable<FilmFacts["audio"]>): boolean {
|
|
96
|
+
return audio.codec === "aac" && audio.sampleRate === FOOTAGE_SAMPLE_RATE && audio.channels === 2;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The final pass: each stream that is already publishable is copied, the other re-encoded, and the `moov` box moved
|
|
101
|
+
* to the front. A film without sound keeps none.
|
|
102
|
+
*/
|
|
103
|
+
export function buildFinishArgs(input: string, facts: Pick<FilmFacts, "video" | "audio">, output: string): string[] {
|
|
104
|
+
const video = isPublishableVideo(facts.video)
|
|
105
|
+
? ["-c:v", "copy"]
|
|
106
|
+
: ["-c:v", "libx264", "-profile:v", "high", "-pix_fmt", "yuv420p", "-preset", "medium", "-crf", String(VIDEO_CRF)];
|
|
107
|
+
const audio =
|
|
108
|
+
facts.audio === null
|
|
109
|
+
? []
|
|
110
|
+
: isPublishableAudio(facts.audio)
|
|
111
|
+
? ["-map", "0:a:0", "-c:a", "copy"]
|
|
112
|
+
: ["-map", "0:a:0", "-c:a", "aac", "-b:a", AUDIO_BITRATE, "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", "2"];
|
|
113
|
+
return ["-v", "error", "-y", "-i", input, "-map", "0:v:0", ...video, ...audio, "-movflags", "+faststart", output];
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** The types of an MP4's top-level boxes, in file order (`ftyp`, `moov`, `mdat`, …). */
|
|
117
|
+
export function readTopLevelBoxes(path: string): string[] {
|
|
118
|
+
const size = statSync(path).size;
|
|
119
|
+
const fd = openSync(path, "r");
|
|
120
|
+
const header = Buffer.alloc(16);
|
|
121
|
+
const types: string[] = [];
|
|
122
|
+
try {
|
|
123
|
+
let offset = 0;
|
|
124
|
+
while (offset + 8 <= size) {
|
|
125
|
+
readSync(fd, header, 0, 16, offset);
|
|
126
|
+
let length = header.readUInt32BE(0);
|
|
127
|
+
types.push(header.toString("latin1", 4, 8));
|
|
128
|
+
// 1: a 64-bit length follows the type; 0: the box runs to the end of the file.
|
|
129
|
+
if (length === 1) length = Number(header.readBigUInt64BE(8));
|
|
130
|
+
if (length === 0) break;
|
|
131
|
+
if (length < 8) throw new Error(`${path}: a box of ${length} bytes at ${offset} is not an MP4 box.`);
|
|
132
|
+
offset += length;
|
|
133
|
+
}
|
|
134
|
+
} finally {
|
|
135
|
+
closeSync(fd);
|
|
136
|
+
}
|
|
137
|
+
return types;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** True when the `moov` box comes before the media (`mdat`), so a player can start before the whole file is in. */
|
|
141
|
+
export function isFastStart(boxes: string[]): boolean {
|
|
142
|
+
const moov = boxes.indexOf("moov");
|
|
143
|
+
const mdat = boxes.indexOf("mdat");
|
|
144
|
+
return moov !== -1 && (mdat === -1 || moov < mdat);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** What keeps a finished film from being publishable; empty when nothing does. */
|
|
148
|
+
export function findFinishProblems(facts: FilmFacts, boxes: string[]): string[] {
|
|
149
|
+
const problems: string[] = [];
|
|
150
|
+
const { video, audio } = facts;
|
|
151
|
+
if (!isPublishableVideo(video)) problems.push(`the picture is ${video.codec} ${video.profile ?? "-"} ${video.pixelFormat ?? "-"}, not h264 High yuv420p`);
|
|
152
|
+
if (audio !== null && !isPublishableAudio(audio)) problems.push(`the sound is ${audio.codec} ${audio.sampleRate} Hz ${audio.channels} channels, not aac 48000 Hz stereo`);
|
|
153
|
+
if (!isFastStart(boxes)) problems.push("the moov box is not before the media (no fast start)");
|
|
154
|
+
return problems;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** The default cover time: the middle of the first talking segment, or of the first segment when nobody talks. */
|
|
158
|
+
export function getDefaultCoverSeconds(edl: Edl, roles: Record<string, ClipRole>): number {
|
|
159
|
+
const starts = getSegmentStarts(edl);
|
|
160
|
+
const index = Math.max(0, edl.segments.findIndex((segment) => roles[segment.clip] === "talk"));
|
|
161
|
+
const segment = edl.segments[index];
|
|
162
|
+
if (segment === undefined) return 0;
|
|
163
|
+
return Math.round(((starts[index] ?? 0) + (segment.end - segment.start) / 2) * 1000) / 1000;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** A cover time inside the film: at most just before its end. */
|
|
167
|
+
export function clampCoverSeconds(seconds: number, durationSeconds: number): number {
|
|
168
|
+
return Math.min(Math.max(0, seconds), Math.max(0, Math.round((durationSeconds - COVER_END_MARGIN) * 1000) / 1000));
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** One frame of the finished film as a JPG. */
|
|
172
|
+
export function buildCoverArgs(film: string, seconds: number, output: string): string[] {
|
|
173
|
+
return ["-v", "error", "-y", "-ss", String(seconds), "-i", film, "-frames:v", "1", "-q:v", "2", output];
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** The cover's path next to the film: its name with `.jpg`. */
|
|
177
|
+
export function getCoverPath(film: string): string {
|
|
178
|
+
const extension = extname(film);
|
|
179
|
+
return `${extension.length > 0 ? film.slice(0, -extension.length) : film}.jpg`;
|
|
180
|
+
}
|
package/src/footage/render.ts
CHANGED
|
@@ -1,10 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
import type { Edl } from "./edl.js";
|
|
1
|
+
import { getSoundSpan, type Edl } from "./edl.js";
|
|
4
2
|
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
5
3
|
import { volumeExpression, type VolumeKeyframe } from "./music.js";
|
|
6
4
|
|
|
7
|
-
/** ffmpeg steps of the footage render: pure argument builders
|
|
5
|
+
/** ffmpeg steps of the footage render: pure argument builders, tested without ffmpeg (`runFfmpeg` in `process.ts` runs them). */
|
|
8
6
|
|
|
9
7
|
/** A short fade at every cut of the speech track, so a cut never clicks. */
|
|
10
8
|
const CUT_FADE_SECONDS = 0.02;
|
|
@@ -13,19 +11,38 @@ const MUSIC_FADE_OUT = 1.5;
|
|
|
13
11
|
|
|
14
12
|
const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
|
|
15
13
|
|
|
16
|
-
/**
|
|
14
|
+
/**
|
|
15
|
+
* The speech track: every segment's sound cut from its sound span (its own clip's, or `audio`'s: a J/L-cut or b-roll
|
|
16
|
+
* over speech) and joined in order; `audio.ownVolume` keeps the picture clip's own sound under it.
|
|
17
|
+
*/
|
|
17
18
|
export function buildSpeechArgs(edl: Edl, sources: Record<string, string>, output: string): string[] {
|
|
18
19
|
const inputs: string[] = [];
|
|
19
20
|
const parts: string[] = [];
|
|
21
|
+
const getSource = (clip: string): string => {
|
|
22
|
+
const source = sources[clip];
|
|
23
|
+
if (source === undefined) throw new Error(`No audio for clip "${clip}" of the EDL.`);
|
|
24
|
+
return source;
|
|
25
|
+
};
|
|
26
|
+
const cut = (input: number, span: { start: number; end: number }, duration: number): string =>
|
|
27
|
+
`[${input}:a]atrim=start=${span.start}:end=${span.end},asetpts=PTS-STARTPTS,aformat=sample_rates=${FOOTAGE_SAMPLE_RATE}:channel_layouts=stereo,apad=whole_dur=${duration}`;
|
|
20
28
|
edl.segments.forEach((segment, index) => {
|
|
21
|
-
const
|
|
22
|
-
if (source === undefined) throw new Error(`No audio for clip "${segment.clip}" of the EDL.`);
|
|
23
|
-
inputs.push("-i", source);
|
|
29
|
+
const sound = getSoundSpan(segment);
|
|
24
30
|
const duration = r3(segment.end - segment.start);
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
31
|
+
const fades = `afade=t=in:d=${CUT_FADE_SECONDS},afade=t=out:st=${r3(Math.max(0, duration - CUT_FADE_SECONDS))}:d=${CUT_FADE_SECONDS}`;
|
|
32
|
+
const input = inputs.length / 2;
|
|
33
|
+
inputs.push("-i", getSource(sound.clip));
|
|
34
|
+
const own = segment.audio?.ownVolume ?? 0;
|
|
35
|
+
// A borrowed span may be a few milliseconds longer than the picture: it is cut to the picture's length.
|
|
36
|
+
const fit = segment.audio === undefined ? "" : `,atrim=end=${duration}`;
|
|
37
|
+
if (own <= 0) {
|
|
38
|
+
parts.push(`${cut(input, sound, duration)}${fit},${fades}[a${index}]`);
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
const ownInput = inputs.length / 2;
|
|
42
|
+
inputs.push("-i", getSource(segment.clip));
|
|
43
|
+
parts.push(`${cut(input, sound, duration)},atrim=end=${duration}[s${index}]`);
|
|
44
|
+
parts.push(`${cut(ownInput, segment, duration)},atrim=end=${duration},volume=${own}[o${index}]`);
|
|
45
|
+
parts.push(`[s${index}][o${index}]amix=inputs=2:duration=first:normalize=0,${fades}[a${index}]`);
|
|
29
46
|
});
|
|
30
47
|
const joined = edl.segments.map((_, index) => `[a${index}]`).join("");
|
|
31
48
|
const graph = `${parts.join(";")};${joined}concat=n=${edl.segments.length}:v=0:a=1[speech]`;
|
|
@@ -73,10 +90,3 @@ export function buildLastFrameArgs(edl: Edl, clips: Record<string, string>, outp
|
|
|
73
90
|
export function buildAudioArgs(clip: string, output: string): string[] {
|
|
74
91
|
return ["-v", "error", "-y", "-i", clip, "-vn", "-c:a", "libmp3lame", "-q:a", "2", output];
|
|
75
92
|
}
|
|
76
|
-
|
|
77
|
-
/** Runs ffmpeg; throws naming the step (a failure here is a broken input or machine, not an expected outcome). */
|
|
78
|
-
export function runFfmpeg(step: string, args: string[]): void {
|
|
79
|
-
const result = spawnSync("ffmpeg", args, { stdio: ["ignore", "inherit", "inherit"] });
|
|
80
|
-
if (result.error !== undefined) throw new Error(`${step}: ffmpeg did not start (${result.error.message}).`);
|
|
81
|
-
if (result.status !== 0) throw new Error(`${step}: ffmpeg ended with code ${String(result.status)} (signal ${result.signal ?? "-"}).`);
|
|
82
|
-
}
|