@softure-ai/marketing-kit 0.3.1 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/README.md +135 -14
- package/dist/cli/footage.d.ts.map +1 -1
- package/dist/cli/footage.js +15 -6
- package/dist/cli/footage.js.map +1 -1
- package/dist/config/actions-schema.d.ts +14 -14
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +5 -0
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +92 -69
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +40 -4
- package/dist/config/schema.js.map +1 -1
- package/dist/config/shot-steps.d.ts +16 -16
- package/dist/footage/audio.d.ts +52 -0
- package/dist/footage/audio.d.ts.map +1 -0
- package/dist/footage/audio.js +100 -0
- package/dist/footage/audio.js.map +1 -0
- package/dist/footage/captions.d.ts +57 -0
- package/dist/footage/captions.d.ts.map +1 -0
- package/dist/footage/captions.js +112 -0
- package/dist/footage/captions.js.map +1 -0
- package/dist/footage/compose.d.ts +5 -5
- package/dist/footage/compose.d.ts.map +1 -1
- package/dist/footage/compose.js +31 -22
- package/dist/footage/compose.js.map +1 -1
- package/dist/footage/edl.d.ts +54 -16
- package/dist/footage/edl.d.ts.map +1 -1
- package/dist/footage/edl.js +204 -52
- package/dist/footage/edl.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts +4 -1
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +18 -6
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/fake.d.ts +2 -0
- package/dist/footage/fake.d.ts.map +1 -1
- package/dist/footage/fake.js +2 -1
- package/dist/footage/fake.js.map +1 -1
- package/dist/footage/film.d.ts +35 -1
- package/dist/footage/film.d.ts.map +1 -1
- package/dist/footage/film.js +9 -1
- package/dist/footage/film.js.map +1 -1
- package/dist/footage/frames.d.ts +38 -0
- package/dist/footage/frames.d.ts.map +1 -0
- package/dist/footage/frames.js +40 -0
- package/dist/footage/frames.js.map +1 -0
- package/dist/footage/loudness.d.ts +49 -0
- package/dist/footage/loudness.d.ts.map +1 -0
- package/dist/footage/loudness.js +114 -0
- package/dist/footage/loudness.js.map +1 -0
- package/dist/footage/normalize.d.ts +6 -2
- package/dist/footage/normalize.d.ts.map +1 -1
- package/dist/footage/normalize.js +7 -9
- package/dist/footage/normalize.js.map +1 -1
- package/dist/footage/paid.d.ts +70 -0
- package/dist/footage/paid.d.ts.map +1 -0
- package/dist/footage/paid.js +172 -0
- package/dist/footage/paid.js.map +1 -0
- package/dist/footage/pipeline.d.ts +34 -10
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +294 -85
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/probe.d.ts +18 -4
- package/dist/footage/probe.d.ts.map +1 -1
- package/dist/footage/probe.js +18 -13
- package/dist/footage/probe.js.map +1 -1
- package/dist/footage/process.d.ts +45 -0
- package/dist/footage/process.d.ts.map +1 -0
- package/dist/footage/process.js +109 -0
- package/dist/footage/process.js.map +1 -0
- package/dist/footage/providers.d.ts +49 -5
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js +4 -0
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/publish.d.ts +62 -0
- package/dist/footage/publish.d.ts.map +1 -0
- package/dist/footage/publish.js +164 -0
- package/dist/footage/publish.js.map +1 -0
- package/dist/footage/render.d.ts +5 -4
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +29 -17
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/sentences.d.ts +44 -0
- package/dist/footage/sentences.d.ts.map +1 -0
- package/dist/footage/sentences.js +102 -0
- package/dist/footage/sentences.js.map +1 -0
- package/dist/footage/tools.d.ts +33 -0
- package/dist/footage/tools.d.ts.map +1 -0
- package/dist/footage/tools.js +108 -0
- package/dist/footage/tools.js.map +1 -0
- package/dist/footage/words.d.ts +6 -0
- package/dist/footage/words.d.ts.map +1 -0
- package/dist/footage/words.js +14 -0
- package/dist/footage/words.js.map +1 -0
- package/dist/index.d.ts +15 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -6
- package/dist/index.js.map +1 -1
- package/docker/footage-worker.Dockerfile +29 -0
- package/package.json +2 -1
- package/schema/edl.schema.json +115 -0
- package/schema/marketing.schema.json +112 -5
- package/src/cli/footage.ts +13 -5
- package/src/config/config.ts +5 -0
- package/src/config/schema.ts +40 -4
- package/src/footage/audio.ts +123 -0
- package/src/footage/captions.ts +146 -0
- package/src/footage/compose.ts +32 -24
- package/src/footage/edl.ts +225 -56
- package/src/footage/elevenlabs.ts +21 -8
- package/src/footage/fake.ts +4 -1
- package/src/footage/film.ts +43 -2
- package/src/footage/frames.ts +65 -0
- package/src/footage/loudness.ts +143 -0
- package/src/footage/normalize.ts +7 -9
- package/src/footage/paid.ts +213 -0
- package/src/footage/pipeline.ts +344 -89
- package/src/footage/probe.ts +20 -12
- package/src/footage/process.ts +135 -0
- package/src/footage/providers.ts +56 -6
- package/src/footage/publish.ts +180 -0
- package/src/footage/render.ts +29 -19
- package/src/footage/sentences.ts +137 -0
- package/src/footage/tools.ts +119 -0
- package/src/footage/words.ts +15 -0
- package/src/index.ts +84 -0
package/dist/footage/pipeline.js
CHANGED
|
@@ -1,20 +1,28 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
1
|
import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
2
|
import { createRequire } from "node:module";
|
|
4
3
|
import { dirname, extname, join, resolve } from "node:path";
|
|
5
4
|
import { LAYOUTS, resolveLayout } from "../compose/timeline.js";
|
|
6
|
-
import {
|
|
5
|
+
import { getHyperframesBin, getHyperframesEnv } from "../render/hyperframes.js";
|
|
7
6
|
import { copyFonts } from "../render/render.js";
|
|
8
7
|
import { err, ok } from "../voice/provider.js";
|
|
9
|
-
import {
|
|
8
|
+
import { checkMusicFileType, describeAudioProbe, probeAudio } from "./audio.js";
|
|
9
|
+
import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
|
|
10
10
|
import { composeFootage, END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
|
|
11
|
-
import {
|
|
11
|
+
import { findMissingCaptionGlyphs, getCaptionPresetLayout } from "./captions.js";
|
|
12
|
+
import { buildEdl, readEdl, remapWords, validateEdl } from "./edl.js";
|
|
12
13
|
import { makePlaceholderTranscript } from "./fake.js";
|
|
14
|
+
import { getFootageEdit } from "./film.js";
|
|
15
|
+
import { DEFAULT_LOUDNESS, normalizeLoudness } from "./loudness.js";
|
|
13
16
|
import { duckingKeyframes } from "./music.js";
|
|
14
17
|
import { normalizeFootage } from "./normalize.js";
|
|
15
18
|
import { describeProbe, getTrimmedDuration, probeFootage } from "./probe.js";
|
|
16
|
-
import {
|
|
17
|
-
import {
|
|
19
|
+
import { getClipCacheKey, getDownloadPath, getEdlPath, getIsolatedCache, getPromptMusicCache, getTranscriptCache, getVideoMusicCache, isUrl, } from "./paid.js";
|
|
20
|
+
import { runFfmpeg, runProcess } from "./process.js";
|
|
21
|
+
import { splitFilmSentences } from "./sentences.js";
|
|
22
|
+
import { buildCoverArgs, buildFinishArgs, clampCoverSeconds, findFinishProblems, getCoverPath, getDefaultCoverSeconds, isPublishableAudio, isPublishableVideo, probeFilmFacts, readTopLevelBoxes, } from "./publish.js";
|
|
23
|
+
import { describePaidSteps, toPaidAudio } from "./providers.js";
|
|
24
|
+
import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs } from "./render.js";
|
|
25
|
+
export { getEdlPath } from "./paid.js";
|
|
18
26
|
/**
|
|
19
27
|
* The footage film end to end, as standalone steps (`prepareFootage` → `transcribeFootage` → `cutFootage` →
|
|
20
28
|
* `renderFootage`) and as one call (`renderFootageFilm`). Paid calls happen only with `isCommit`; without it a step
|
|
@@ -23,7 +31,15 @@ import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs
|
|
|
23
31
|
* film's (`placeholder/` in the build folder, its own EDL).
|
|
24
32
|
*
|
|
25
33
|
* Build folder layout: `sources/` (downloaded URLs), `clips/<id>.mp4` (normalized), `audio/<id>.mp3`, `edl.json`
|
|
26
|
-
* (`edl.placeholder.json`), `assets/`, `index.html
|
|
34
|
+
* (`edl.placeholder.json`), `assets/`, `index.html`, `render.mp4` (hyperframes' output until the final pass).
|
|
35
|
+
*
|
|
36
|
+
* The finished film is publishable: the speech track normalized to a loudness target before the music goes under it,
|
|
37
|
+
* H.264 High, AAC 48 kHz stereo, fast start, a cover JPG next to it, and its facts read back from the written file.
|
|
38
|
+
*
|
|
39
|
+
* Every step spawns ffmpeg, ffprobe and hyperframes asynchronously, so a jobs worker's event loop (and its lease
|
|
40
|
+
* heartbeat) keeps running. `signal` stops a run: the child process is killed, its partial file removed, and the
|
|
41
|
+
* step rejects with the signal's reason. `onProgress` reports each stage in seconds of media, `onCharge` each paid
|
|
42
|
+
* call made.
|
|
27
43
|
*/
|
|
28
44
|
const require = createRequire(import.meta.url);
|
|
29
45
|
/** Bumped when the normalized format changes, so older normalized clips are made again. */
|
|
@@ -31,22 +47,33 @@ const NORMALIZE_VERSION = 1;
|
|
|
31
47
|
export function getFrame(film) {
|
|
32
48
|
return LAYOUTS[film.format].frame;
|
|
33
49
|
}
|
|
34
|
-
|
|
35
|
-
|
|
50
|
+
/** Reports one stage's progress over its items, each `seconds` long; the reporter takes an item's index and its seconds done. */
|
|
51
|
+
function createStageProgress(onProgress, step, items) {
|
|
52
|
+
const total = items.reduce((sum, item) => sum + item.seconds, 0);
|
|
53
|
+
const offsets = items.map((_, index) => items.slice(0, index).reduce((sum, item) => sum + item.seconds, 0));
|
|
54
|
+
return (index, seconds) => {
|
|
55
|
+
const entry = items[index];
|
|
56
|
+
if (onProgress === undefined || entry === undefined)
|
|
57
|
+
return;
|
|
58
|
+
const done = (offsets[index] ?? 0) + Math.min(Math.max(seconds, 0), entry.seconds);
|
|
59
|
+
onProgress({ step, item: entry.item, itemIndex: index + 1, itemCount: items.length, done: Math.round(done * 1000) / 1000, total: Math.round(total * 1000) / 1000 });
|
|
60
|
+
};
|
|
36
61
|
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
62
|
+
/** Rejects with the signal's reason once the run was aborted (before a paid call, which the provider may not stop). */
|
|
63
|
+
function throwIfAborted(signal) {
|
|
64
|
+
signal?.throwIfAborted();
|
|
65
|
+
}
|
|
66
|
+
async function download(url, dir, options) {
|
|
67
|
+
const file = getDownloadPath(url, dir);
|
|
42
68
|
if (existsSync(file))
|
|
43
69
|
return ok(file);
|
|
44
|
-
log(`download: ${url}`);
|
|
70
|
+
options.log(`download: ${url}`);
|
|
45
71
|
let response;
|
|
46
72
|
try {
|
|
47
|
-
response = await fetch(url);
|
|
73
|
+
response = await (options.fetch ?? globalThis.fetch)(url, { signal: options.signal ?? null });
|
|
48
74
|
}
|
|
49
75
|
catch (error) {
|
|
76
|
+
throwIfAborted(options.signal);
|
|
50
77
|
return err(`downloading ${url} failed: ${String(error)}`);
|
|
51
78
|
}
|
|
52
79
|
if (!response.ok)
|
|
@@ -62,9 +89,9 @@ function getNormalizeStamp(clip, source, frame) {
|
|
|
62
89
|
const stat = statSync(source);
|
|
63
90
|
return JSON.stringify({ version: NORMALIZE_VERSION, source: resolve(source), size: stat.size, mtime: stat.mtimeMs, trim: clip.trim, fit: clip.fit, frame });
|
|
64
91
|
}
|
|
65
|
-
/** Downloads URL clips, probes every clip (all problems at once), normalizes each
|
|
92
|
+
/** Downloads URL clips, probes every clip (all problems at once) and a local music file, normalizes each clip that changed. */
|
|
66
93
|
export async function prepareFootage(film, options) {
|
|
67
|
-
const { buildDir, log } = options;
|
|
94
|
+
const { buildDir, log, signal } = options;
|
|
68
95
|
if (film.clips.length === 0)
|
|
69
96
|
return err(`footage film ${film.id} has no clips.`);
|
|
70
97
|
const sources = {};
|
|
@@ -75,39 +102,53 @@ export async function prepareFootage(film, options) {
|
|
|
75
102
|
sources[clip.id] = clip.source;
|
|
76
103
|
continue;
|
|
77
104
|
}
|
|
78
|
-
const downloaded = await download(clip.source, join(buildDir, "sources"), options
|
|
105
|
+
const downloaded = await download(clip.source, join(buildDir, "sources"), options);
|
|
79
106
|
if (!downloaded.ok)
|
|
80
107
|
return err(`clip "${clip.id}": ${downloaded.error}`);
|
|
81
108
|
sources[clip.id] = downloaded.value;
|
|
82
109
|
}
|
|
83
|
-
const probed = probeFootage(film.clips, sources);
|
|
110
|
+
const probed = await probeFootage(film.clips, sources, signal === undefined ? {} : { signal });
|
|
84
111
|
if (!probed.ok)
|
|
85
112
|
return err(probed.errors.join("\n"));
|
|
113
|
+
if (film.music.source.kind === "file") {
|
|
114
|
+
const music = await checkMusicFile(film.music.source.path, options);
|
|
115
|
+
if (!music.ok)
|
|
116
|
+
return err(music.error);
|
|
117
|
+
}
|
|
86
118
|
const frame = getFrame(film);
|
|
87
119
|
const clipsDir = join(buildDir, "clips");
|
|
88
120
|
mkdirSync(clipsDir, { recursive: true });
|
|
89
121
|
const prepared = [];
|
|
122
|
+
const stale = [];
|
|
90
123
|
for (const { clip, file: source, probe } of probed.clips) {
|
|
91
124
|
log(describeProbe(clip, probe));
|
|
92
125
|
const file = join(clipsDir, `${clip.id}.mp4`);
|
|
93
126
|
const stampPath = join(clipsDir, `${clip.id}.json`);
|
|
94
127
|
const stamp = getNormalizeStamp(clip, source, frame);
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
128
|
+
const entry = { clip, source, probe, file, duration: getTrimmedDuration(clip, probe) };
|
|
129
|
+
if (!existsSync(file) || !existsSync(stampPath) || readFileSync(stampPath, "utf8") !== stamp)
|
|
130
|
+
stale.push({ clip: entry, stamp, stampPath });
|
|
131
|
+
prepared.push(entry);
|
|
132
|
+
}
|
|
133
|
+
const progress = createStageProgress(options.onProgress, "normalize", stale.map(({ clip }) => ({ item: clip.clip.id, seconds: clip.duration })));
|
|
134
|
+
for (const [index, { clip, stamp, stampPath }] of stale.entries()) {
|
|
135
|
+
progress(index, 0);
|
|
136
|
+
// The stamp goes first, so a normalize stopped half-way never leaves a stamp that vouches for a partial clip.
|
|
137
|
+
rmSync(stampPath, { force: true });
|
|
138
|
+
await normalizeFootage({ clip: clip.clip, file: clip.source, probe: clip.probe, frame, output: clip.file }, { ...(signal === undefined ? {} : { signal }), onSeconds: (seconds) => progress(index, seconds) });
|
|
139
|
+
writeFileSync(stampPath, stamp);
|
|
140
|
+
progress(index, clip.duration);
|
|
100
141
|
}
|
|
101
142
|
return ok(prepared);
|
|
102
143
|
}
|
|
103
144
|
/** The cache identity of a clip's source part: its bytes and its trim. */
|
|
104
|
-
|
|
105
|
-
return
|
|
145
|
+
function getClipKey(clip) {
|
|
146
|
+
return getClipCacheKey(clip.source, clip.clip);
|
|
106
147
|
}
|
|
107
|
-
function extractAudio(clip, buildDir) {
|
|
148
|
+
async function extractAudio(clip, buildDir, signal) {
|
|
108
149
|
const audio = join(buildDir, "audio", `${clip.clip.id}.mp3`);
|
|
109
150
|
mkdirSync(dirname(audio), { recursive: true });
|
|
110
|
-
runFfmpeg(`Extracting the sound of clip ${clip.clip.id}`, buildAudioArgs(clip.file, audio));
|
|
151
|
+
await runFfmpeg(`Extracting the sound of clip ${clip.clip.id}`, buildAudioArgs(clip.file, audio), { ...(signal === undefined ? {} : { signal }), output: audio });
|
|
111
152
|
return audio;
|
|
112
153
|
}
|
|
113
154
|
/** The free transcripts of `--placeholder`, written under `placeholder/` for a look. */
|
|
@@ -126,23 +167,17 @@ export function getMusicPaidStep(film, options, seconds) {
|
|
|
126
167
|
const source = film.music.source;
|
|
127
168
|
if (source.kind === "none" || source.kind === "file")
|
|
128
169
|
return null;
|
|
129
|
-
if (source.kind === "prompt" && existsSync(
|
|
170
|
+
if (source.kind === "prompt" && existsSync(getPromptMusicCache(film, options).path))
|
|
130
171
|
return null;
|
|
131
172
|
return { kind: "music", item: "film", provider: options.providers.music.id, seconds };
|
|
132
173
|
}
|
|
133
|
-
function getPromptMusicPath(film, options) {
|
|
134
|
-
const source = film.music.source;
|
|
135
|
-
const prompt = source.kind === "prompt" ? source.prompt : "";
|
|
136
|
-
const key = footageCacheKey("music", { prompt, provider: options.providers.music.id, model: options.providers.music.model });
|
|
137
|
-
return getFootageCachePath(options.cacheDir, film.id, "music", key);
|
|
138
|
-
}
|
|
139
174
|
/**
|
|
140
175
|
* Each talking clip's words with times, from the cache or (with `isCommit`) from the speech-to-text provider, the
|
|
141
176
|
* voice cleaned first when the film isolates it. Without `isCommit`, a cache miss returns the dry run: every paid
|
|
142
177
|
* call the film still needs, music included, with its length.
|
|
143
178
|
*/
|
|
144
179
|
export async function transcribeFootage(film, clips, options) {
|
|
145
|
-
const { buildDir,
|
|
180
|
+
const { buildDir, providers, log, signal } = options;
|
|
146
181
|
if (options.isPlaceholder)
|
|
147
182
|
return ok(transcribePlaceholder(film, clips, buildDir));
|
|
148
183
|
const plan = [];
|
|
@@ -151,12 +186,11 @@ export async function transcribeFootage(film, clips, options) {
|
|
|
151
186
|
if (clip.clip.role !== "talk")
|
|
152
187
|
continue;
|
|
153
188
|
const clipKey = await getClipKey(clip);
|
|
154
|
-
const isolated =
|
|
155
|
-
const
|
|
156
|
-
|
|
157
|
-
if (isolated !== null && !existsSync(isolated))
|
|
189
|
+
const isolated = getIsolatedCache(film, clipKey, options);
|
|
190
|
+
const transcript = getTranscriptCache(film, clipKey, options);
|
|
191
|
+
if (isolated !== null && !existsSync(isolated.path))
|
|
158
192
|
steps.push({ kind: "isolate", item: `clip ${clip.clip.id}`, provider: providers.isolation.id, seconds: clip.duration });
|
|
159
|
-
if (!existsSync(transcript))
|
|
193
|
+
if (!existsSync(transcript.path))
|
|
160
194
|
steps.push({ kind: "transcribe", item: `clip ${clip.clip.id}`, provider: providers.stt.id, seconds: clip.duration });
|
|
161
195
|
plan.push({ clip, isolated, transcript });
|
|
162
196
|
}
|
|
@@ -175,24 +209,37 @@ export async function transcribeFootage(film, clips, options) {
|
|
|
175
209
|
words[clip.clip.id] = [];
|
|
176
210
|
speech[clip.clip.id] = clip.file;
|
|
177
211
|
}
|
|
178
|
-
|
|
212
|
+
const isolating = plan.filter(({ isolated }) => isolated !== null && !existsSync(isolated.path));
|
|
213
|
+
const transcribing = plan.filter(({ transcript }) => !existsSync(transcript.path));
|
|
214
|
+
const isolateProgress = createStageProgress(options.onProgress, "isolate", isolating.map(({ clip }) => ({ item: clip.clip.id, seconds: clip.duration })));
|
|
215
|
+
const transcribeProgress = createStageProgress(options.onProgress, "transcribe", transcribing.map(({ clip }) => ({ item: clip.clip.id, seconds: clip.duration })));
|
|
216
|
+
for (const entry of plan) {
|
|
217
|
+
const { clip, isolated, transcript } = entry;
|
|
179
218
|
const id = clip.clip.id;
|
|
180
219
|
let audio = null;
|
|
181
|
-
const getAudio = () => (audio ??= extractAudio(clip, buildDir));
|
|
220
|
+
const getAudio = () => (audio ??= extractAudio(clip, buildDir, signal));
|
|
182
221
|
let sttInput = null;
|
|
183
222
|
if (isolated !== null) {
|
|
184
|
-
if (!existsSync(isolated)) {
|
|
223
|
+
if (!existsSync(isolated.path)) {
|
|
224
|
+
const index = isolating.indexOf(entry);
|
|
225
|
+
isolateProgress(index, 0);
|
|
226
|
+
const input = await getAudio();
|
|
227
|
+
throwIfAborted(signal);
|
|
185
228
|
log(`voice isolation (${providers.isolation.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
|
|
186
|
-
const cleaned = await providers.isolation.isolate({ audio:
|
|
229
|
+
const cleaned = await providers.isolation.isolate({ audio: input, ...(signal === undefined ? {} : { signal }) });
|
|
230
|
+
throwIfAborted(signal);
|
|
187
231
|
if (!cleaned.ok)
|
|
188
232
|
return err(`clip "${id}": ${cleaned.error}`);
|
|
189
|
-
|
|
190
|
-
|
|
233
|
+
const paid = toPaidAudio(cleaned.value);
|
|
234
|
+
writeCacheFile(isolated.path, paid.audio);
|
|
235
|
+
options.onCharge?.({ step: "isolate", item: id, provider: providers.isolation.id, units: clip.duration, ...(paid.credits === undefined ? {} : { credits: paid.credits }), cacheKey: isolated.key });
|
|
236
|
+
log(`voice isolation: saved ${isolated.path} (commit it).`);
|
|
237
|
+
isolateProgress(index, clip.duration);
|
|
191
238
|
}
|
|
192
|
-
speech[id] = isolated;
|
|
193
|
-
sttInput = isolated;
|
|
239
|
+
speech[id] = isolated.path;
|
|
240
|
+
sttInput = isolated.path;
|
|
194
241
|
}
|
|
195
|
-
const cached = readCachedTranscript(transcript);
|
|
242
|
+
const cached = readCachedTranscript(transcript.path);
|
|
196
243
|
if (!cached.ok)
|
|
197
244
|
return err(cached.error);
|
|
198
245
|
if (cached.words !== null) {
|
|
@@ -202,23 +249,36 @@ export async function transcribeFootage(film, clips, options) {
|
|
|
202
249
|
}
|
|
203
250
|
if (sttInput !== null) {
|
|
204
251
|
// The provider sees the clip's id as the file name, as it does for the plain sound.
|
|
252
|
+
const cleaned = sttInput;
|
|
205
253
|
sttInput = join(buildDir, "audio", `${id}.isolated.mp3`);
|
|
206
254
|
mkdirSync(dirname(sttInput), { recursive: true });
|
|
207
|
-
copyFileSync(
|
|
255
|
+
copyFileSync(cleaned, sttInput);
|
|
208
256
|
}
|
|
257
|
+
const index = transcribing.indexOf(entry);
|
|
258
|
+
transcribeProgress(index, 0);
|
|
259
|
+
const input = sttInput ?? (await getAudio());
|
|
260
|
+
throwIfAborted(signal);
|
|
209
261
|
log(`speech to text (${providers.stt.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
|
|
210
|
-
const result = await providers.stt.transcribe({ audio:
|
|
262
|
+
const result = await providers.stt.transcribe({ audio: input, language: film.language, ...(signal === undefined ? {} : { signal }) });
|
|
263
|
+
throwIfAborted(signal);
|
|
211
264
|
if (!result.ok)
|
|
212
265
|
return err(`clip "${id}": ${result.error}`);
|
|
213
|
-
writeTranscript(transcript, result.value.words);
|
|
214
|
-
|
|
266
|
+
writeTranscript(transcript.path, result.value.words);
|
|
267
|
+
const { credits } = result.value;
|
|
268
|
+
options.onCharge?.({ step: "transcribe", item: id, provider: providers.stt.id, units: clip.duration, ...(credits === undefined ? {} : { credits }), cacheKey: transcript.key });
|
|
269
|
+
log(`transcript: clip ${id}, ${result.value.words.length} words, saved ${transcript.path} (commit it).`);
|
|
270
|
+
transcribeProgress(index, clip.duration);
|
|
215
271
|
words[id] = result.value.words;
|
|
216
272
|
}
|
|
217
273
|
return ok({ kind: "ready", words, speech });
|
|
218
274
|
}
|
|
219
|
-
/** The
|
|
220
|
-
export function
|
|
221
|
-
|
|
275
|
+
/** The talking clips' sentences with their ids (`s1`…), as `buildEdl` names them, for choosing what to keep. */
|
|
276
|
+
export function getFootageSentences(clips, words) {
|
|
277
|
+
return splitFilmSentences(clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, words: words[clip.clip.id] ?? [] })));
|
|
278
|
+
}
|
|
279
|
+
/** The cut from the transcripts, written to the EDL file (replacing what is there); `keepSentences` keeps only those sentences. */
|
|
280
|
+
export function cutFootage(film, clips, words, edlPath, options = {}) {
|
|
281
|
+
const edl = buildEdl(film.id, clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration, words: words[clip.clip.id] ?? [] })), film.cut, film.transitions, { isKenBurns: getFootageEdit(film).kenBurnsPercent > 0, ...(options.keepSentences === undefined ? {} : { keepSentences: options.keepSentences }) });
|
|
222
282
|
mkdirSync(dirname(edlPath), { recursive: true });
|
|
223
283
|
writeFileSync(edlPath, `${JSON.stringify(edl, null, 2)}\n`);
|
|
224
284
|
return edl;
|
|
@@ -235,9 +295,22 @@ export function readEdlFile(film, clips, edlPath) {
|
|
|
235
295
|
const read = readEdl(json);
|
|
236
296
|
if (!read.ok)
|
|
237
297
|
return err(`${edlPath}: ${read.error}`);
|
|
238
|
-
const problems =
|
|
298
|
+
const problems = validateEdl(read.edl, clips.map((clip) => ({ id: clip.clip.id, duration: clip.duration })), film.id);
|
|
239
299
|
return problems.length > 0 ? err(`${edlPath}: ${problems.join("; ")}`) : ok(read.edl);
|
|
240
300
|
}
|
|
301
|
+
/** A local music file checked like an app's upload: an accepted type, then readable audio (`probeAudio`). */
|
|
302
|
+
async function checkMusicFile(path, options) {
|
|
303
|
+
if (!existsSync(path))
|
|
304
|
+
return err(`music: no file at ${path}`);
|
|
305
|
+
const type = checkMusicFileType({ fileName: path });
|
|
306
|
+
if (!type.ok)
|
|
307
|
+
return err(`music: ${type.error}`);
|
|
308
|
+
const probe = await probeAudio(path, options.signal === undefined ? {} : { signal: options.signal });
|
|
309
|
+
if (!probe.ok)
|
|
310
|
+
return err(`music: ${probe.error}`);
|
|
311
|
+
options.log(`music: ${path} (${describeAudioProbe(probe.probe)}).`);
|
|
312
|
+
return ok(path);
|
|
313
|
+
}
|
|
241
314
|
/**
|
|
242
315
|
* The music file for the cut: a local file as it is, generated music from the cache, or (with `isCommit`) bought
|
|
243
316
|
* from the music provider and cached. Null when the film has none or the rehearsal skips generated music; a paid
|
|
@@ -251,18 +324,18 @@ async function getMusic(film, edl, clips, seconds, options) {
|
|
|
251
324
|
return existsSync(source.path) ? ok(source.path) : err(`music: no file at ${source.path}`);
|
|
252
325
|
if (options.isPlaceholder)
|
|
253
326
|
return ok(null);
|
|
254
|
-
const { providers, log } = options;
|
|
255
|
-
let
|
|
327
|
+
const { providers, log, signal } = options;
|
|
328
|
+
let cache;
|
|
256
329
|
if (source.kind === "prompt") {
|
|
257
|
-
|
|
330
|
+
cache = getPromptMusicCache(film, options);
|
|
258
331
|
}
|
|
259
332
|
else {
|
|
260
333
|
const clipKeys = {};
|
|
261
334
|
for (const clip of clips)
|
|
262
335
|
clipKeys[clip.clip.id] = await getClipKey(clip);
|
|
263
|
-
|
|
264
|
-
path = getFootageCachePath(options.cacheDir, film.id, "music", key);
|
|
336
|
+
cache = getVideoMusicCache(film, edl, clipKeys, options);
|
|
265
337
|
}
|
|
338
|
+
const { path } = cache;
|
|
266
339
|
if (existsSync(path)) {
|
|
267
340
|
log(`music: from the cache ${path}, nothing spent.`);
|
|
268
341
|
return ok(path);
|
|
@@ -270,25 +343,36 @@ async function getMusic(film, edl, clips, seconds, options) {
|
|
|
270
343
|
const step = { kind: "music", item: "film", provider: providers.music.id, seconds };
|
|
271
344
|
if (!options.isCommit)
|
|
272
345
|
return ok(step);
|
|
273
|
-
|
|
346
|
+
const progress = createStageProgress(options.onProgress, "music", [{ item: "film", seconds }]);
|
|
347
|
+
progress(0, 0);
|
|
348
|
+
const aborting = signal === undefined ? {} : { signal };
|
|
274
349
|
let bought;
|
|
275
350
|
if (source.kind === "prompt") {
|
|
276
|
-
|
|
351
|
+
throwIfAborted(signal);
|
|
352
|
+
log(`music (${providers.music.id}): ${seconds.toFixed(1)} s.`);
|
|
353
|
+
bought = await providers.music.compose({ prompt: source.prompt, seconds, ...aborting });
|
|
277
354
|
}
|
|
278
355
|
else {
|
|
279
356
|
const preview = join(options.buildDir, "cut-preview.mp4");
|
|
280
|
-
|
|
281
|
-
|
|
357
|
+
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file]));
|
|
358
|
+
await runFfmpeg("The cut preview for Video to Music", buildCutPreviewArgs(edl, clipFiles, preview), { ...aborting, output: preview });
|
|
359
|
+
throwIfAborted(signal);
|
|
360
|
+
log(`music (${providers.music.id}): ${seconds.toFixed(1)} s.`);
|
|
361
|
+
bought = await providers.music.scoreVideo({ video: preview, description: source.description, tags: source.tags, ...aborting });
|
|
282
362
|
}
|
|
363
|
+
throwIfAborted(signal);
|
|
283
364
|
if (!bought.ok)
|
|
284
365
|
return err(`music: ${bought.error}`);
|
|
285
|
-
|
|
366
|
+
const paid = toPaidAudio(bought.value);
|
|
367
|
+
writeCacheFile(path, paid.audio);
|
|
368
|
+
options.onCharge?.({ step: "music", item: "film", provider: providers.music.id, units: seconds, ...(paid.credits === undefined ? {} : { credits: paid.credits }), cacheKey: cache.key });
|
|
286
369
|
log(`music: saved ${path} (commit it).`);
|
|
370
|
+
progress(0, seconds);
|
|
287
371
|
return ok(path);
|
|
288
372
|
}
|
|
289
|
-
/** Composes the cut and renders it: speech track, ducked music, end card backdrop, assets, hyperframes. */
|
|
373
|
+
/** Composes the cut and renders it: speech track (loudness-normalized), ducked music, end card backdrop, assets, hyperframes, final pass, cover. */
|
|
290
374
|
export async function renderFootage(film, clips, transcript, edl, options) {
|
|
291
|
-
const { buildDir, log } = options;
|
|
375
|
+
const { buildDir, log, signal } = options;
|
|
292
376
|
const seconds = getFootageFilmSeconds(film, edl);
|
|
293
377
|
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
294
378
|
const music = await getMusic(film, edl, clips, seconds, options);
|
|
@@ -304,14 +388,42 @@ export async function renderFootage(film, clips, transcript, edl, options) {
|
|
|
304
388
|
rmSync(assetsDir, { recursive: true, force: true });
|
|
305
389
|
mkdirSync(assetsDir, { recursive: true });
|
|
306
390
|
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file]));
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
391
|
+
const musicFile = typeof music.value === "string" ? music.value : null;
|
|
392
|
+
const aborting = signal === undefined ? {} : { signal };
|
|
393
|
+
const loudness = film.loudness === undefined ? DEFAULT_LOUDNESS : film.loudness;
|
|
394
|
+
const speechFile = join(assetsDir, "speech.mp3");
|
|
395
|
+
const rawSpeechFile = loudness === null ? speechFile : join(assetsDir, "speech.raw.mp3");
|
|
396
|
+
const ffmpegTrack = (label, output, args) => (onSeconds) => runFfmpeg(label, args(), { ...aborting, output, onSeconds });
|
|
397
|
+
const tracks = [
|
|
398
|
+
{ item: "speech", seconds, run: ffmpegTrack("The speech track", rawSpeechFile, () => buildSpeechArgs(edl, transcript.speech, rawSpeechFile)) },
|
|
399
|
+
];
|
|
400
|
+
if (loudness !== null) {
|
|
401
|
+
tracks.push({
|
|
402
|
+
item: "loudness",
|
|
403
|
+
seconds,
|
|
404
|
+
run: async (onSeconds) => {
|
|
405
|
+
const outcome = await normalizeLoudness(rawSpeechFile, speechFile, loudness, { ...aborting, onSeconds });
|
|
406
|
+
log(outcome.kind === "silent"
|
|
407
|
+
? "loudness: the speech track is silent, left as it is."
|
|
408
|
+
: `loudness: the speech track measured ${outcome.measured.toFixed(1)} LUFS, normalized to ${loudness.target} LUFS (true peak ${loudness.truePeak} dBTP).`);
|
|
409
|
+
},
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
if (musicFile !== null) {
|
|
310
413
|
const keyframes = duckingKeyframes(words, { volume: film.music.volume, duckedVolume: film.music.duckedVolume });
|
|
311
|
-
|
|
312
|
-
|
|
414
|
+
const output = join(assetsDir, "music.mp3");
|
|
415
|
+
tracks.push({ item: "music", seconds, run: ffmpegTrack("The music track", output, () => buildMusicArgs({ input: musicFile, seconds, keyframes, output })) });
|
|
416
|
+
}
|
|
417
|
+
// A single frame: counted as a moment of work so the stage's bar still ends at its total.
|
|
418
|
+
const lastFrame = join(assetsDir, "last.jpg");
|
|
419
|
+
tracks.push({ item: "last frame", seconds: 0.1, run: ffmpegTrack("The end card's backdrop", lastFrame, () => buildLastFrameArgs(edl, clipFiles, lastFrame)) });
|
|
420
|
+
const assetProgress = createStageProgress(options.onProgress, "assets", tracks);
|
|
421
|
+
for (const [index, track] of tracks.entries()) {
|
|
422
|
+
assetProgress(index, 0);
|
|
423
|
+
await track.run((done) => assetProgress(index, done));
|
|
424
|
+
assetProgress(index, track.seconds);
|
|
313
425
|
}
|
|
314
|
-
|
|
426
|
+
const musicAsset = musicFile === null ? null : "assets/music.mp3";
|
|
315
427
|
const fonts = copyFonts(options.brand.fonts, join(assetsDir, "fonts"));
|
|
316
428
|
if (options.brand.logo !== null)
|
|
317
429
|
cpSync(options.brand.logo, join(assetsDir, "logo.svg"));
|
|
@@ -323,7 +435,19 @@ export async function renderFootage(film, clips, transcript, edl, options) {
|
|
|
323
435
|
sfx[event] = `assets/sfx/${name}`;
|
|
324
436
|
}
|
|
325
437
|
cpSync(require.resolve("gsap/dist/gsap.min.js"), join(assetsDir, "gsap.min.js"));
|
|
326
|
-
const
|
|
438
|
+
const base = resolveLayout(film.format, options.layout ?? {});
|
|
439
|
+
// A caption preset places the 9:16 box in a platform's safe zone; the project's own caption override still wins.
|
|
440
|
+
const preset = film.captionPreset === undefined || film.format !== "9:16" ? null : getCaptionPresetLayout(film.captionPreset, base.caption.fontSize);
|
|
441
|
+
const layout = preset === null || options.layout?.caption !== undefined ? base : { ...base, caption: preset };
|
|
442
|
+
const missing = findMissingCaptionGlyphs({
|
|
443
|
+
fonts: options.brand.fonts,
|
|
444
|
+
captions: film.captions ? words.map((word) => word.text) : [],
|
|
445
|
+
endCard: film.endCard,
|
|
446
|
+
brandName: options.brand.name,
|
|
447
|
+
});
|
|
448
|
+
if (missing.length > 0) {
|
|
449
|
+
return err(`the brand fonts cannot draw every letter of the film: ${missing.map((entry) => `${entry.font} lacks ${entry.characters.join(" ")}`).join("; ")}. Add a font file that covers them (brand.fonts).`);
|
|
450
|
+
}
|
|
327
451
|
const html = composeFootage({
|
|
328
452
|
film,
|
|
329
453
|
edl,
|
|
@@ -347,12 +471,97 @@ export async function renderFootage(film, clips, transcript, edl, options) {
|
|
|
347
471
|
writeFileSync(join(buildDir, "index.html"), html);
|
|
348
472
|
mkdirSync(dirname(options.output), { recursive: true });
|
|
349
473
|
log(`render: ${edl.segments.length} segments, ${seconds.toFixed(1)} s.`);
|
|
350
|
-
const
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
474
|
+
const renderFile = resolve(buildDir, "render.mp4");
|
|
475
|
+
const rendered = await runHyperframesRender(options, renderFile, seconds);
|
|
476
|
+
if (!rendered.ok)
|
|
477
|
+
return rendered;
|
|
478
|
+
const roles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.clip.role]));
|
|
479
|
+
const finished = await finishFootage(film, { renderFile, seconds, coverAt: film.cover?.at ?? getDefaultCoverSeconds(edl, roles) }, options);
|
|
480
|
+
if (!finished.ok)
|
|
481
|
+
return finished;
|
|
482
|
+
return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder), ...finished.value });
|
|
483
|
+
}
|
|
484
|
+
/**
|
|
485
|
+
* The final pass over hyperframes' render: publishable codecs and fast start (stream copy where the render already
|
|
486
|
+
* matches), checked with ffprobe, then the cover and the file's facts.
|
|
487
|
+
*/
|
|
488
|
+
async function finishFootage(film, input, options) {
|
|
489
|
+
const { log, signal, output } = options;
|
|
490
|
+
const aborting = signal === undefined ? {} : { signal };
|
|
491
|
+
const progress = createStageProgress(options.onProgress, "finish", [{ item: "film", seconds: input.seconds }]);
|
|
492
|
+
progress(0, 0);
|
|
493
|
+
const rendered = await probeFilmFacts(input.renderFile, aborting);
|
|
494
|
+
if (!rendered.ok)
|
|
495
|
+
return err(`the rendered film: ${rendered.error}`);
|
|
496
|
+
const isCopied = isPublishableVideo(rendered.facts.video) && (rendered.facts.audio === null || isPublishableAudio(rendered.facts.audio));
|
|
497
|
+
await runFfmpeg("The film's final pass", buildFinishArgs(input.renderFile, rendered.facts, output), { ...aborting, output, onSeconds: (done) => progress(0, done) });
|
|
498
|
+
rmSync(input.renderFile, { force: true });
|
|
499
|
+
const finished = await probeFilmFacts(output, aborting);
|
|
500
|
+
if (!finished.ok)
|
|
501
|
+
return err(`the finished film: ${finished.error}`);
|
|
502
|
+
const problems = findFinishProblems(finished.facts, readTopLevelBoxes(output));
|
|
503
|
+
if (problems.length > 0)
|
|
504
|
+
return err(`the finished film ${output} is not publishable: ${problems.join("; ")}.`);
|
|
505
|
+
progress(0, input.seconds);
|
|
506
|
+
const { bytes, width, height, durationSeconds } = finished.facts;
|
|
507
|
+
log(`finish: ${isCopied ? "streams copied" : "re-encoded"} to H.264 High, AAC 48 kHz stereo, fast start; ${width}×${height}, ${durationSeconds.toFixed(2)} s, ${bytes} bytes.`);
|
|
508
|
+
let cover = null;
|
|
509
|
+
if (film.cover !== null) {
|
|
510
|
+
cover = getCoverPath(output);
|
|
511
|
+
const at = clampCoverSeconds(input.coverAt, durationSeconds);
|
|
512
|
+
if (at !== input.coverAt)
|
|
513
|
+
log(`cover: ${input.coverAt} s is past the film's end, the frame at ${at} s is used.`);
|
|
514
|
+
await runFfmpeg("The cover frame", buildCoverArgs(output, at, cover), { ...aborting, output: cover });
|
|
515
|
+
log(`cover: ${cover} (at ${at} s).`);
|
|
516
|
+
}
|
|
517
|
+
return ok({ bytes, width, height, durationSeconds, cover });
|
|
518
|
+
}
|
|
519
|
+
/** hyperframes' progress lines carry a percentage (`… 42%`); the last one on a line counts. */
|
|
520
|
+
export function readRenderPercent(line) {
|
|
521
|
+
const matches = [...line.matchAll(/(\d{1,3}(?:\.\d+)?)\s?%/g)];
|
|
522
|
+
const last = matches.at(-1)?.[1];
|
|
523
|
+
if (last === undefined)
|
|
524
|
+
return null;
|
|
525
|
+
const percent = Number(last);
|
|
526
|
+
return percent >= 0 && percent <= 100 ? percent : null;
|
|
527
|
+
}
|
|
528
|
+
/** The hyperframes render as a child process: its lines go to the log (progress every 10 %), its percentages to `onProgress`. */
|
|
529
|
+
async function runHyperframesRender(options, output, seconds) {
|
|
530
|
+
const progress = createStageProgress(options.onProgress, "render", [{ item: "film", seconds }]);
|
|
531
|
+
progress(0, 0);
|
|
532
|
+
let loggedTenth = -1;
|
|
533
|
+
const onLine = (line) => {
|
|
534
|
+
if (line.trim().length === 0)
|
|
535
|
+
return;
|
|
536
|
+
const percent = readRenderPercent(line);
|
|
537
|
+
if (percent === null) {
|
|
538
|
+
options.log(line.trim());
|
|
539
|
+
return;
|
|
540
|
+
}
|
|
541
|
+
progress(0, (percent / 100) * seconds);
|
|
542
|
+
// A progress line per frame would flood the log: one line per tenth of the render is kept.
|
|
543
|
+
const tenth = Math.floor(percent / 10);
|
|
544
|
+
if (tenth > loggedTenth) {
|
|
545
|
+
loggedTenth = tenth;
|
|
546
|
+
options.log(line.trim());
|
|
547
|
+
}
|
|
548
|
+
};
|
|
549
|
+
const result = await runProcess({
|
|
550
|
+
command: process.execPath,
|
|
551
|
+
args: [getHyperframesBin(), "render", "--quality", options.quality, "-o", output],
|
|
552
|
+
cwd: options.buildDir,
|
|
553
|
+
env: getHyperframesEnv(),
|
|
554
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
555
|
+
removeOnAbort: [output],
|
|
556
|
+
onStdoutLine: onLine,
|
|
557
|
+
onStderrLine: onLine,
|
|
558
|
+
});
|
|
559
|
+
if (result.kind === "not-started")
|
|
560
|
+
return err(`hyperframes render did not start: ${result.error}.`);
|
|
561
|
+
if (result.code !== 0)
|
|
562
|
+
return err(`hyperframes render ended with code ${String(result.code)} (signal ${result.signal ?? "-"}).`);
|
|
563
|
+
progress(0, seconds);
|
|
564
|
+
return ok(null);
|
|
356
565
|
}
|
|
357
566
|
/**
|
|
358
567
|
* The whole footage film from a `FootageFilm` object, no `marketing.json` needed: prepare, transcribe (or the dry
|
|
@@ -377,7 +586,7 @@ export async function renderFootageFilm(film, options) {
|
|
|
377
586
|
options.log(`cut: ${edlPath} (${edl.segments.length} segments), as it is; softure-marketing footage --step=cut builds it again.`);
|
|
378
587
|
}
|
|
379
588
|
else {
|
|
380
|
-
edl = cutFootage(film, clips.value, transcript.value.words, edlPath);
|
|
589
|
+
edl = cutFootage(film, clips.value, transcript.value.words, edlPath, options.keepSentences === undefined ? {} : { keepSentences: options.keepSentences });
|
|
381
590
|
options.log(`cut: ${edl.segments.length} segments written to ${edlPath}.`);
|
|
382
591
|
}
|
|
383
592
|
return renderFootage(film, clips.value, transcript.value, edl, options);
|