@softure-ai/marketing-kit 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/CHANGELOG.md +57 -0
  2. package/README.md +135 -14
  3. package/dist/cli/footage.d.ts.map +1 -1
  4. package/dist/cli/footage.js +15 -6
  5. package/dist/cli/footage.js.map +1 -1
  6. package/dist/config/actions-schema.d.ts +14 -14
  7. package/dist/config/config.d.ts.map +1 -1
  8. package/dist/config/config.js +5 -0
  9. package/dist/config/config.js.map +1 -1
  10. package/dist/config/schema.d.ts +92 -69
  11. package/dist/config/schema.d.ts.map +1 -1
  12. package/dist/config/schema.js +40 -4
  13. package/dist/config/schema.js.map +1 -1
  14. package/dist/config/shot-steps.d.ts +16 -16
  15. package/dist/footage/audio.d.ts +52 -0
  16. package/dist/footage/audio.d.ts.map +1 -0
  17. package/dist/footage/audio.js +100 -0
  18. package/dist/footage/audio.js.map +1 -0
  19. package/dist/footage/captions.d.ts +57 -0
  20. package/dist/footage/captions.d.ts.map +1 -0
  21. package/dist/footage/captions.js +112 -0
  22. package/dist/footage/captions.js.map +1 -0
  23. package/dist/footage/compose.d.ts +5 -5
  24. package/dist/footage/compose.d.ts.map +1 -1
  25. package/dist/footage/compose.js +31 -22
  26. package/dist/footage/compose.js.map +1 -1
  27. package/dist/footage/edl.d.ts +54 -16
  28. package/dist/footage/edl.d.ts.map +1 -1
  29. package/dist/footage/edl.js +204 -52
  30. package/dist/footage/edl.js.map +1 -1
  31. package/dist/footage/elevenlabs.d.ts +4 -1
  32. package/dist/footage/elevenlabs.d.ts.map +1 -1
  33. package/dist/footage/elevenlabs.js +18 -6
  34. package/dist/footage/elevenlabs.js.map +1 -1
  35. package/dist/footage/fake.d.ts +2 -0
  36. package/dist/footage/fake.d.ts.map +1 -1
  37. package/dist/footage/fake.js +2 -1
  38. package/dist/footage/fake.js.map +1 -1
  39. package/dist/footage/film.d.ts +35 -1
  40. package/dist/footage/film.d.ts.map +1 -1
  41. package/dist/footage/film.js +9 -1
  42. package/dist/footage/film.js.map +1 -1
  43. package/dist/footage/frames.d.ts +38 -0
  44. package/dist/footage/frames.d.ts.map +1 -0
  45. package/dist/footage/frames.js +40 -0
  46. package/dist/footage/frames.js.map +1 -0
  47. package/dist/footage/loudness.d.ts +49 -0
  48. package/dist/footage/loudness.d.ts.map +1 -0
  49. package/dist/footage/loudness.js +114 -0
  50. package/dist/footage/loudness.js.map +1 -0
  51. package/dist/footage/normalize.d.ts +6 -2
  52. package/dist/footage/normalize.d.ts.map +1 -1
  53. package/dist/footage/normalize.js +7 -9
  54. package/dist/footage/normalize.js.map +1 -1
  55. package/dist/footage/paid.d.ts +70 -0
  56. package/dist/footage/paid.d.ts.map +1 -0
  57. package/dist/footage/paid.js +172 -0
  58. package/dist/footage/paid.js.map +1 -0
  59. package/dist/footage/pipeline.d.ts +34 -10
  60. package/dist/footage/pipeline.d.ts.map +1 -1
  61. package/dist/footage/pipeline.js +294 -85
  62. package/dist/footage/pipeline.js.map +1 -1
  63. package/dist/footage/probe.d.ts +18 -4
  64. package/dist/footage/probe.d.ts.map +1 -1
  65. package/dist/footage/probe.js +18 -13
  66. package/dist/footage/probe.js.map +1 -1
  67. package/dist/footage/process.d.ts +45 -0
  68. package/dist/footage/process.d.ts.map +1 -0
  69. package/dist/footage/process.js +109 -0
  70. package/dist/footage/process.js.map +1 -0
  71. package/dist/footage/providers.d.ts +49 -5
  72. package/dist/footage/providers.d.ts.map +1 -1
  73. package/dist/footage/providers.js +4 -0
  74. package/dist/footage/providers.js.map +1 -1
  75. package/dist/footage/publish.d.ts +62 -0
  76. package/dist/footage/publish.d.ts.map +1 -0
  77. package/dist/footage/publish.js +164 -0
  78. package/dist/footage/publish.js.map +1 -0
  79. package/dist/footage/render.d.ts +5 -4
  80. package/dist/footage/render.d.ts.map +1 -1
  81. package/dist/footage/render.js +29 -17
  82. package/dist/footage/render.js.map +1 -1
  83. package/dist/footage/sentences.d.ts +44 -0
  84. package/dist/footage/sentences.d.ts.map +1 -0
  85. package/dist/footage/sentences.js +102 -0
  86. package/dist/footage/sentences.js.map +1 -0
  87. package/dist/footage/tools.d.ts +33 -0
  88. package/dist/footage/tools.d.ts.map +1 -0
  89. package/dist/footage/tools.js +108 -0
  90. package/dist/footage/tools.js.map +1 -0
  91. package/dist/footage/words.d.ts +6 -0
  92. package/dist/footage/words.d.ts.map +1 -0
  93. package/dist/footage/words.js +14 -0
  94. package/dist/footage/words.js.map +1 -0
  95. package/dist/index.d.ts +15 -6
  96. package/dist/index.d.ts.map +1 -1
  97. package/dist/index.js +15 -6
  98. package/dist/index.js.map +1 -1
  99. package/docker/footage-worker.Dockerfile +29 -0
  100. package/package.json +2 -1
  101. package/schema/edl.schema.json +115 -0
  102. package/schema/marketing.schema.json +112 -5
  103. package/src/cli/footage.ts +13 -5
  104. package/src/config/config.ts +5 -0
  105. package/src/config/schema.ts +40 -4
  106. package/src/footage/audio.ts +123 -0
  107. package/src/footage/captions.ts +146 -0
  108. package/src/footage/compose.ts +32 -24
  109. package/src/footage/edl.ts +225 -56
  110. package/src/footage/elevenlabs.ts +21 -8
  111. package/src/footage/fake.ts +4 -1
  112. package/src/footage/film.ts +43 -2
  113. package/src/footage/frames.ts +65 -0
  114. package/src/footage/loudness.ts +143 -0
  115. package/src/footage/normalize.ts +7 -9
  116. package/src/footage/paid.ts +213 -0
  117. package/src/footage/pipeline.ts +344 -89
  118. package/src/footage/probe.ts +20 -12
  119. package/src/footage/process.ts +135 -0
  120. package/src/footage/providers.ts +56 -6
  121. package/src/footage/publish.ts +180 -0
  122. package/src/footage/render.ts +29 -19
  123. package/src/footage/sentences.ts +137 -0
  124. package/src/footage/tools.ts +119 -0
  125. package/src/footage/words.ts +15 -0
  126. package/src/index.ts +84 -0
@@ -1,20 +1,28 @@
1
- import { createHash } from "node:crypto";
2
1
  import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
3
2
  import { createRequire } from "node:module";
4
3
  import { dirname, extname, join, resolve } from "node:path";
5
4
  import { LAYOUTS, resolveLayout } from "../compose/timeline.js";
6
- import { runHyperframes } from "../render/hyperframes.js";
5
+ import { getHyperframesBin, getHyperframesEnv } from "../render/hyperframes.js";
7
6
  import { copyFonts } from "../render/render.js";
8
7
  import { err, ok } from "../voice/provider.js";
9
- import { footageCacheKey, getFootageCachePath, hashFile, readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
8
+ import { checkMusicFileType, describeAudioProbe, probeAudio } from "./audio.js";
9
+ import { readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
10
10
  import { composeFootage, END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
11
- import { buildEdl, findEdlProblems, readEdl, remapWords } from "./edl.js";
11
+ import { findMissingCaptionGlyphs, getCaptionPresetLayout } from "./captions.js";
12
+ import { buildEdl, readEdl, remapWords, validateEdl } from "./edl.js";
12
13
  import { makePlaceholderTranscript } from "./fake.js";
14
+ import { getFootageEdit } from "./film.js";
15
+ import { DEFAULT_LOUDNESS, normalizeLoudness } from "./loudness.js";
13
16
  import { duckingKeyframes } from "./music.js";
14
17
  import { normalizeFootage } from "./normalize.js";
15
18
  import { describeProbe, getTrimmedDuration, probeFootage } from "./probe.js";
16
- import { describePaidSteps } from "./providers.js";
17
- import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs, runFfmpeg } from "./render.js";
19
+ import { getClipCacheKey, getDownloadPath, getEdlPath, getIsolatedCache, getPromptMusicCache, getTranscriptCache, getVideoMusicCache, isUrl, } from "./paid.js";
20
+ import { runFfmpeg, runProcess } from "./process.js";
21
+ import { splitFilmSentences } from "./sentences.js";
22
+ import { buildCoverArgs, buildFinishArgs, clampCoverSeconds, findFinishProblems, getCoverPath, getDefaultCoverSeconds, isPublishableAudio, isPublishableVideo, probeFilmFacts, readTopLevelBoxes, } from "./publish.js";
23
+ import { describePaidSteps, toPaidAudio } from "./providers.js";
24
+ import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs } from "./render.js";
25
+ export { getEdlPath } from "./paid.js";
18
26
  /**
19
27
  * The footage film end to end, as standalone steps (`prepareFootage` → `transcribeFootage` → `cutFootage` →
20
28
  * `renderFootage`) and as one call (`renderFootageFilm`). Paid calls happen only with `isCommit`; without it a step
@@ -23,7 +31,15 @@ import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs
23
31
  * film's (`placeholder/` in the build folder, its own EDL).
24
32
  *
25
33
  * Build folder layout: `sources/` (downloaded URLs), `clips/<id>.mp4` (normalized), `audio/<id>.mp3`, `edl.json`
26
- * (`edl.placeholder.json`), `assets/`, `index.html`.
34
+ * (`edl.placeholder.json`), `assets/`, `index.html`, `render.mp4` (hyperframes' output until the final pass).
35
+ *
36
+ * The finished film is publishable: the speech track normalized to a loudness target before the music goes under it,
37
+ * H.264 High, AAC 48 kHz stereo, fast start, a cover JPG next to it, and its facts read back from the written file.
38
+ *
39
+ * Every step spawns ffmpeg, ffprobe and hyperframes asynchronously, so a jobs worker's event loop (and its lease
40
+ * heartbeat) keeps running. `signal` stops a run: the child process is killed, its partial file removed, and the
41
+ * step rejects with the signal's reason. `onProgress` reports each stage in seconds of media, `onCharge` each paid
42
+ * call made.
27
43
  */
28
44
  const require = createRequire(import.meta.url);
29
45
  /** Bumped when the normalized format changes, so older normalized clips are made again. */
@@ -31,22 +47,33 @@ const NORMALIZE_VERSION = 1;
31
47
  export function getFrame(film) {
32
48
  return LAYOUTS[film.format].frame;
33
49
  }
34
- export function getEdlPath(buildDir, isPlaceholder) {
35
- return join(buildDir, isPlaceholder ? "edl.placeholder.json" : "edl.json");
50
+ /** Reports one stage's progress over its items, each `seconds` long; the reporter takes an item's index and its seconds done. */
51
+ function createStageProgress(onProgress, step, items) {
52
+ const total = items.reduce((sum, item) => sum + item.seconds, 0);
53
+ const offsets = items.map((_, index) => items.slice(0, index).reduce((sum, item) => sum + item.seconds, 0));
54
+ return (index, seconds) => {
55
+ const entry = items[index];
56
+ if (onProgress === undefined || entry === undefined)
57
+ return;
58
+ const done = (offsets[index] ?? 0) + Math.min(Math.max(seconds, 0), entry.seconds);
59
+ onProgress({ step, item: entry.item, itemIndex: index + 1, itemCount: items.length, done: Math.round(done * 1000) / 1000, total: Math.round(total * 1000) / 1000 });
60
+ };
36
61
  }
37
- const isUrl = (source) => /^https?:\/\//i.test(source);
38
- async function download(url, dir, fetch, log) {
39
- const name = createHash("sha256").update(url).digest("hex").slice(0, 16);
40
- const extension = extname(new URL(url).pathname).toLowerCase() || ".mp4";
41
- const file = join(dir, `${name}${extension}`);
62
+ /** Rejects with the signal's reason once the run was aborted (before a paid call, which the provider may not stop). */
63
+ function throwIfAborted(signal) {
64
+ signal?.throwIfAborted();
65
+ }
66
+ async function download(url, dir, options) {
67
+ const file = getDownloadPath(url, dir);
42
68
  if (existsSync(file))
43
69
  return ok(file);
44
- log(`download: ${url}`);
70
+ options.log(`download: ${url}`);
45
71
  let response;
46
72
  try {
47
- response = await fetch(url);
73
+ response = await (options.fetch ?? globalThis.fetch)(url, { signal: options.signal ?? null });
48
74
  }
49
75
  catch (error) {
76
+ throwIfAborted(options.signal);
50
77
  return err(`downloading ${url} failed: ${String(error)}`);
51
78
  }
52
79
  if (!response.ok)
@@ -62,9 +89,9 @@ function getNormalizeStamp(clip, source, frame) {
62
89
  const stat = statSync(source);
63
90
  return JSON.stringify({ version: NORMALIZE_VERSION, source: resolve(source), size: stat.size, mtime: stat.mtimeMs, trim: clip.trim, fit: clip.fit, frame });
64
91
  }
65
- /** Downloads URL clips, probes every clip (all problems at once), normalizes each one that changed. */
92
+ /** Downloads URL clips, probes every clip (all problems at once) and a local music file, normalizes each clip that changed. */
66
93
  export async function prepareFootage(film, options) {
67
- const { buildDir, log } = options;
94
+ const { buildDir, log, signal } = options;
68
95
  if (film.clips.length === 0)
69
96
  return err(`footage film ${film.id} has no clips.`);
70
97
  const sources = {};
@@ -75,39 +102,53 @@ export async function prepareFootage(film, options) {
75
102
  sources[clip.id] = clip.source;
76
103
  continue;
77
104
  }
78
- const downloaded = await download(clip.source, join(buildDir, "sources"), options.fetch ?? globalThis.fetch, log);
105
+ const downloaded = await download(clip.source, join(buildDir, "sources"), options);
79
106
  if (!downloaded.ok)
80
107
  return err(`clip "${clip.id}": ${downloaded.error}`);
81
108
  sources[clip.id] = downloaded.value;
82
109
  }
83
- const probed = probeFootage(film.clips, sources);
110
+ const probed = await probeFootage(film.clips, sources, signal === undefined ? {} : { signal });
84
111
  if (!probed.ok)
85
112
  return err(probed.errors.join("\n"));
113
+ if (film.music.source.kind === "file") {
114
+ const music = await checkMusicFile(film.music.source.path, options);
115
+ if (!music.ok)
116
+ return err(music.error);
117
+ }
86
118
  const frame = getFrame(film);
87
119
  const clipsDir = join(buildDir, "clips");
88
120
  mkdirSync(clipsDir, { recursive: true });
89
121
  const prepared = [];
122
+ const stale = [];
90
123
  for (const { clip, file: source, probe } of probed.clips) {
91
124
  log(describeProbe(clip, probe));
92
125
  const file = join(clipsDir, `${clip.id}.mp4`);
93
126
  const stampPath = join(clipsDir, `${clip.id}.json`);
94
127
  const stamp = getNormalizeStamp(clip, source, frame);
95
- if (!existsSync(file) || !existsSync(stampPath) || readFileSync(stampPath, "utf8") !== stamp) {
96
- normalizeFootage({ clip, file: source, probe, frame, output: file });
97
- writeFileSync(stampPath, stamp);
98
- }
99
- prepared.push({ clip, source, probe, file, duration: getTrimmedDuration(clip, probe) });
128
+ const entry = { clip, source, probe, file, duration: getTrimmedDuration(clip, probe) };
129
+ if (!existsSync(file) || !existsSync(stampPath) || readFileSync(stampPath, "utf8") !== stamp)
130
+ stale.push({ clip: entry, stamp, stampPath });
131
+ prepared.push(entry);
132
+ }
133
+ const progress = createStageProgress(options.onProgress, "normalize", stale.map(({ clip }) => ({ item: clip.clip.id, seconds: clip.duration })));
134
+ for (const [index, { clip, stamp, stampPath }] of stale.entries()) {
135
+ progress(index, 0);
136
+ // The stamp goes first, so a normalize stopped half-way never leaves a stamp that vouches for a partial clip.
137
+ rmSync(stampPath, { force: true });
138
+ await normalizeFootage({ clip: clip.clip, file: clip.source, probe: clip.probe, frame, output: clip.file }, { ...(signal === undefined ? {} : { signal }), onSeconds: (seconds) => progress(index, seconds) });
139
+ writeFileSync(stampPath, stamp);
140
+ progress(index, clip.duration);
100
141
  }
101
142
  return ok(prepared);
102
143
  }
103
144
  /** The cache identity of a clip's source part: its bytes and its trim. */
104
- async function getClipKey(clip) {
105
- return footageCacheKey("transcript", { source: await hashFile(clip.source), trim: clip.clip.trim });
145
+ function getClipKey(clip) {
146
+ return getClipCacheKey(clip.source, clip.clip);
106
147
  }
107
- function extractAudio(clip, buildDir) {
148
+ async function extractAudio(clip, buildDir, signal) {
108
149
  const audio = join(buildDir, "audio", `${clip.clip.id}.mp3`);
109
150
  mkdirSync(dirname(audio), { recursive: true });
110
- runFfmpeg(`Extracting the sound of clip ${clip.clip.id}`, buildAudioArgs(clip.file, audio));
151
+ await runFfmpeg(`Extracting the sound of clip ${clip.clip.id}`, buildAudioArgs(clip.file, audio), { ...(signal === undefined ? {} : { signal }), output: audio });
111
152
  return audio;
112
153
  }
113
154
  /** The free transcripts of `--placeholder`, written under `placeholder/` for a look. */
@@ -126,23 +167,17 @@ export function getMusicPaidStep(film, options, seconds) {
126
167
  const source = film.music.source;
127
168
  if (source.kind === "none" || source.kind === "file")
128
169
  return null;
129
- if (source.kind === "prompt" && existsSync(getPromptMusicPath(film, options)))
170
+ if (source.kind === "prompt" && existsSync(getPromptMusicCache(film, options).path))
130
171
  return null;
131
172
  return { kind: "music", item: "film", provider: options.providers.music.id, seconds };
132
173
  }
133
- function getPromptMusicPath(film, options) {
134
- const source = film.music.source;
135
- const prompt = source.kind === "prompt" ? source.prompt : "";
136
- const key = footageCacheKey("music", { prompt, provider: options.providers.music.id, model: options.providers.music.model });
137
- return getFootageCachePath(options.cacheDir, film.id, "music", key);
138
- }
139
174
  /**
140
175
  * Each talking clip's words with times, from the cache or (with `isCommit`) from the speech-to-text provider, the
141
176
  * voice cleaned first when the film isolates it. Without `isCommit`, a cache miss returns the dry run: every paid
142
177
  * call the film still needs, music included, with its length.
143
178
  */
144
179
  export async function transcribeFootage(film, clips, options) {
145
- const { buildDir, cacheDir, providers, log } = options;
180
+ const { buildDir, providers, log, signal } = options;
146
181
  if (options.isPlaceholder)
147
182
  return ok(transcribePlaceholder(film, clips, buildDir));
148
183
  const plan = [];
@@ -151,12 +186,11 @@ export async function transcribeFootage(film, clips, options) {
151
186
  if (clip.clip.role !== "talk")
152
187
  continue;
153
188
  const clipKey = await getClipKey(clip);
154
- const isolated = film.isolateVoice ? getFootageCachePath(cacheDir, film.id, "isolated", footageCacheKey("isolated", { clip: clipKey, provider: providers.isolation.id })) : null;
155
- const transcriptKey = footageCacheKey("transcript", { clip: clipKey, isolated: isolated !== null, provider: providers.stt.id, model: providers.stt.model, language: film.language });
156
- const transcript = getFootageCachePath(cacheDir, film.id, "transcript", transcriptKey);
157
- if (isolated !== null && !existsSync(isolated))
189
+ const isolated = getIsolatedCache(film, clipKey, options);
190
+ const transcript = getTranscriptCache(film, clipKey, options);
191
+ if (isolated !== null && !existsSync(isolated.path))
158
192
  steps.push({ kind: "isolate", item: `clip ${clip.clip.id}`, provider: providers.isolation.id, seconds: clip.duration });
159
- if (!existsSync(transcript))
193
+ if (!existsSync(transcript.path))
160
194
  steps.push({ kind: "transcribe", item: `clip ${clip.clip.id}`, provider: providers.stt.id, seconds: clip.duration });
161
195
  plan.push({ clip, isolated, transcript });
162
196
  }
@@ -175,24 +209,37 @@ export async function transcribeFootage(film, clips, options) {
175
209
  words[clip.clip.id] = [];
176
210
  speech[clip.clip.id] = clip.file;
177
211
  }
178
- for (const { clip, isolated, transcript } of plan) {
212
+ const isolating = plan.filter(({ isolated }) => isolated !== null && !existsSync(isolated.path));
213
+ const transcribing = plan.filter(({ transcript }) => !existsSync(transcript.path));
214
+ const isolateProgress = createStageProgress(options.onProgress, "isolate", isolating.map(({ clip }) => ({ item: clip.clip.id, seconds: clip.duration })));
215
+ const transcribeProgress = createStageProgress(options.onProgress, "transcribe", transcribing.map(({ clip }) => ({ item: clip.clip.id, seconds: clip.duration })));
216
+ for (const entry of plan) {
217
+ const { clip, isolated, transcript } = entry;
179
218
  const id = clip.clip.id;
180
219
  let audio = null;
181
- const getAudio = () => (audio ??= extractAudio(clip, buildDir));
220
+ const getAudio = () => (audio ??= extractAudio(clip, buildDir, signal));
182
221
  let sttInput = null;
183
222
  if (isolated !== null) {
184
- if (!existsSync(isolated)) {
223
+ if (!existsSync(isolated.path)) {
224
+ const index = isolating.indexOf(entry);
225
+ isolateProgress(index, 0);
226
+ const input = await getAudio();
227
+ throwIfAborted(signal);
185
228
  log(`voice isolation (${providers.isolation.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
186
- const cleaned = await providers.isolation.isolate({ audio: getAudio() });
229
+ const cleaned = await providers.isolation.isolate({ audio: input, ...(signal === undefined ? {} : { signal }) });
230
+ throwIfAborted(signal);
187
231
  if (!cleaned.ok)
188
232
  return err(`clip "${id}": ${cleaned.error}`);
189
- writeCacheFile(isolated, cleaned.value);
190
- log(`voice isolation: saved ${isolated} (commit it).`);
233
+ const paid = toPaidAudio(cleaned.value);
234
+ writeCacheFile(isolated.path, paid.audio);
235
+ options.onCharge?.({ step: "isolate", item: id, provider: providers.isolation.id, units: clip.duration, ...(paid.credits === undefined ? {} : { credits: paid.credits }), cacheKey: isolated.key });
236
+ log(`voice isolation: saved ${isolated.path} (commit it).`);
237
+ isolateProgress(index, clip.duration);
191
238
  }
192
- speech[id] = isolated;
193
- sttInput = isolated;
239
+ speech[id] = isolated.path;
240
+ sttInput = isolated.path;
194
241
  }
195
- const cached = readCachedTranscript(transcript);
242
+ const cached = readCachedTranscript(transcript.path);
196
243
  if (!cached.ok)
197
244
  return err(cached.error);
198
245
  if (cached.words !== null) {
@@ -202,23 +249,36 @@ export async function transcribeFootage(film, clips, options) {
202
249
  }
203
250
  if (sttInput !== null) {
204
251
  // The provider sees the clip's id as the file name, as it does for the plain sound.
252
+ const cleaned = sttInput;
205
253
  sttInput = join(buildDir, "audio", `${id}.isolated.mp3`);
206
254
  mkdirSync(dirname(sttInput), { recursive: true });
207
- copyFileSync(isolated, sttInput);
255
+ copyFileSync(cleaned, sttInput);
208
256
  }
257
+ const index = transcribing.indexOf(entry);
258
+ transcribeProgress(index, 0);
259
+ const input = sttInput ?? (await getAudio());
260
+ throwIfAborted(signal);
209
261
  log(`speech to text (${providers.stt.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
210
- const result = await providers.stt.transcribe({ audio: sttInput ?? getAudio(), language: film.language });
262
+ const result = await providers.stt.transcribe({ audio: input, language: film.language, ...(signal === undefined ? {} : { signal }) });
263
+ throwIfAborted(signal);
211
264
  if (!result.ok)
212
265
  return err(`clip "${id}": ${result.error}`);
213
- writeTranscript(transcript, result.value.words);
214
- log(`transcript: clip ${id}, ${result.value.words.length} words, saved ${transcript} (commit it).`);
266
+ writeTranscript(transcript.path, result.value.words);
267
+ const { credits } = result.value;
268
+ options.onCharge?.({ step: "transcribe", item: id, provider: providers.stt.id, units: clip.duration, ...(credits === undefined ? {} : { credits }), cacheKey: transcript.key });
269
+ log(`transcript: clip ${id}, ${result.value.words.length} words, saved ${transcript.path} (commit it).`);
270
+ transcribeProgress(index, clip.duration);
215
271
  words[id] = result.value.words;
216
272
  }
217
273
  return ok({ kind: "ready", words, speech });
218
274
  }
219
- /** The cut from the transcripts, written to the EDL file (replacing what is there). */
220
- export function cutFootage(film, clips, words, edlPath) {
221
- const edl = buildEdl(film.id, clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration, words: words[clip.clip.id] ?? [] })), film.cut, film.transitions);
275
+ /** The talking clips' sentences with their ids (`s1`…), as `buildEdl` names them, for choosing what to keep. */
276
+ export function getFootageSentences(clips, words) {
277
+ return splitFilmSentences(clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, words: words[clip.clip.id] ?? [] })));
278
+ }
279
+ /** The cut from the transcripts, written to the EDL file (replacing what is there); `keepSentences` keeps only those sentences. */
280
+ export function cutFootage(film, clips, words, edlPath, options = {}) {
281
+ const edl = buildEdl(film.id, clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration, words: words[clip.clip.id] ?? [] })), film.cut, film.transitions, { isKenBurns: getFootageEdit(film).kenBurnsPercent > 0, ...(options.keepSentences === undefined ? {} : { keepSentences: options.keepSentences }) });
222
282
  mkdirSync(dirname(edlPath), { recursive: true });
223
283
  writeFileSync(edlPath, `${JSON.stringify(edl, null, 2)}\n`);
224
284
  return edl;
@@ -235,9 +295,22 @@ export function readEdlFile(film, clips, edlPath) {
235
295
  const read = readEdl(json);
236
296
  if (!read.ok)
237
297
  return err(`${edlPath}: ${read.error}`);
238
- const problems = findEdlProblems(read.edl, clips.map((clip) => ({ id: clip.clip.id, duration: clip.duration })), film.id);
298
+ const problems = validateEdl(read.edl, clips.map((clip) => ({ id: clip.clip.id, duration: clip.duration })), film.id);
239
299
  return problems.length > 0 ? err(`${edlPath}: ${problems.join("; ")}`) : ok(read.edl);
240
300
  }
301
+ /** A local music file checked like an app's upload: an accepted type, then readable audio (`probeAudio`). */
302
+ async function checkMusicFile(path, options) {
303
+ if (!existsSync(path))
304
+ return err(`music: no file at ${path}`);
305
+ const type = checkMusicFileType({ fileName: path });
306
+ if (!type.ok)
307
+ return err(`music: ${type.error}`);
308
+ const probe = await probeAudio(path, options.signal === undefined ? {} : { signal: options.signal });
309
+ if (!probe.ok)
310
+ return err(`music: ${probe.error}`);
311
+ options.log(`music: ${path} (${describeAudioProbe(probe.probe)}).`);
312
+ return ok(path);
313
+ }
241
314
  /**
242
315
  * The music file for the cut: a local file as it is, generated music from the cache, or (with `isCommit`) bought
243
316
  * from the music provider and cached. Null when the film has none or the rehearsal skips generated music; a paid
@@ -251,18 +324,18 @@ async function getMusic(film, edl, clips, seconds, options) {
251
324
  return existsSync(source.path) ? ok(source.path) : err(`music: no file at ${source.path}`);
252
325
  if (options.isPlaceholder)
253
326
  return ok(null);
254
- const { providers, log } = options;
255
- let path;
327
+ const { providers, log, signal } = options;
328
+ let cache;
256
329
  if (source.kind === "prompt") {
257
- path = getPromptMusicPath(film, options);
330
+ cache = getPromptMusicCache(film, options);
258
331
  }
259
332
  else {
260
333
  const clipKeys = {};
261
334
  for (const clip of clips)
262
335
  clipKeys[clip.clip.id] = await getClipKey(clip);
263
- const key = footageCacheKey("music", { segments: edl.segments, clips: clipKeys, description: source.description, tags: source.tags, provider: providers.music.id, model: providers.music.model });
264
- path = getFootageCachePath(options.cacheDir, film.id, "music", key);
336
+ cache = getVideoMusicCache(film, edl, clipKeys, options);
265
337
  }
338
+ const { path } = cache;
266
339
  if (existsSync(path)) {
267
340
  log(`music: from the cache ${path}, nothing spent.`);
268
341
  return ok(path);
@@ -270,25 +343,36 @@ async function getMusic(film, edl, clips, seconds, options) {
270
343
  const step = { kind: "music", item: "film", provider: providers.music.id, seconds };
271
344
  if (!options.isCommit)
272
345
  return ok(step);
273
- log(`music (${providers.music.id}): ${seconds.toFixed(1)} s.`);
346
+ const progress = createStageProgress(options.onProgress, "music", [{ item: "film", seconds }]);
347
+ progress(0, 0);
348
+ const aborting = signal === undefined ? {} : { signal };
274
349
  let bought;
275
350
  if (source.kind === "prompt") {
276
- bought = await providers.music.compose({ prompt: source.prompt, seconds });
351
+ throwIfAborted(signal);
352
+ log(`music (${providers.music.id}): ${seconds.toFixed(1)} s.`);
353
+ bought = await providers.music.compose({ prompt: source.prompt, seconds, ...aborting });
277
354
  }
278
355
  else {
279
356
  const preview = join(options.buildDir, "cut-preview.mp4");
280
- runFfmpeg("The cut preview for Video to Music", buildCutPreviewArgs(edl, Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file])), preview));
281
- bought = await providers.music.scoreVideo({ video: preview, description: source.description, tags: source.tags });
357
+ const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file]));
358
+ await runFfmpeg("The cut preview for Video to Music", buildCutPreviewArgs(edl, clipFiles, preview), { ...aborting, output: preview });
359
+ throwIfAborted(signal);
360
+ log(`music (${providers.music.id}): ${seconds.toFixed(1)} s.`);
361
+ bought = await providers.music.scoreVideo({ video: preview, description: source.description, tags: source.tags, ...aborting });
282
362
  }
363
+ throwIfAborted(signal);
283
364
  if (!bought.ok)
284
365
  return err(`music: ${bought.error}`);
285
- writeCacheFile(path, bought.value);
366
+ const paid = toPaidAudio(bought.value);
367
+ writeCacheFile(path, paid.audio);
368
+ options.onCharge?.({ step: "music", item: "film", provider: providers.music.id, units: seconds, ...(paid.credits === undefined ? {} : { credits: paid.credits }), cacheKey: cache.key });
286
369
  log(`music: saved ${path} (commit it).`);
370
+ progress(0, seconds);
287
371
  return ok(path);
288
372
  }
289
- /** Composes the cut and renders it: speech track, ducked music, end card backdrop, assets, hyperframes. */
373
+ /** Composes the cut and renders it: speech track (loudness-normalized), ducked music, end card backdrop, assets, hyperframes, final pass, cover. */
290
374
  export async function renderFootage(film, clips, transcript, edl, options) {
291
- const { buildDir, log } = options;
375
+ const { buildDir, log, signal } = options;
292
376
  const seconds = getFootageFilmSeconds(film, edl);
293
377
  const words = remapWords(edl, transcript.words, film.cut.fillers);
294
378
  const music = await getMusic(film, edl, clips, seconds, options);
@@ -304,14 +388,42 @@ export async function renderFootage(film, clips, transcript, edl, options) {
304
388
  rmSync(assetsDir, { recursive: true, force: true });
305
389
  mkdirSync(assetsDir, { recursive: true });
306
390
  const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file]));
307
- runFfmpeg("The speech track", buildSpeechArgs(edl, transcript.speech, join(assetsDir, "speech.mp3")));
308
- let musicAsset = null;
309
- if (typeof music.value === "string") {
391
+ const musicFile = typeof music.value === "string" ? music.value : null;
392
+ const aborting = signal === undefined ? {} : { signal };
393
+ const loudness = film.loudness === undefined ? DEFAULT_LOUDNESS : film.loudness;
394
+ const speechFile = join(assetsDir, "speech.mp3");
395
+ const rawSpeechFile = loudness === null ? speechFile : join(assetsDir, "speech.raw.mp3");
396
+ const ffmpegTrack = (label, output, args) => (onSeconds) => runFfmpeg(label, args(), { ...aborting, output, onSeconds });
397
+ const tracks = [
398
+ { item: "speech", seconds, run: ffmpegTrack("The speech track", rawSpeechFile, () => buildSpeechArgs(edl, transcript.speech, rawSpeechFile)) },
399
+ ];
400
+ if (loudness !== null) {
401
+ tracks.push({
402
+ item: "loudness",
403
+ seconds,
404
+ run: async (onSeconds) => {
405
+ const outcome = await normalizeLoudness(rawSpeechFile, speechFile, loudness, { ...aborting, onSeconds });
406
+ log(outcome.kind === "silent"
407
+ ? "loudness: the speech track is silent, left as it is."
408
+ : `loudness: the speech track measured ${outcome.measured.toFixed(1)} LUFS, normalized to ${loudness.target} LUFS (true peak ${loudness.truePeak} dBTP).`);
409
+ },
410
+ });
411
+ }
412
+ if (musicFile !== null) {
310
413
  const keyframes = duckingKeyframes(words, { volume: film.music.volume, duckedVolume: film.music.duckedVolume });
311
- runFfmpeg("The music track", buildMusicArgs({ input: music.value, seconds, keyframes, output: join(assetsDir, "music.mp3") }));
312
- musicAsset = "assets/music.mp3";
414
+ const output = join(assetsDir, "music.mp3");
415
+ tracks.push({ item: "music", seconds, run: ffmpegTrack("The music track", output, () => buildMusicArgs({ input: musicFile, seconds, keyframes, output })) });
416
+ }
417
+ // A single frame: counted as a moment of work so the stage's bar still ends at its total.
418
+ const lastFrame = join(assetsDir, "last.jpg");
419
+ tracks.push({ item: "last frame", seconds: 0.1, run: ffmpegTrack("The end card's backdrop", lastFrame, () => buildLastFrameArgs(edl, clipFiles, lastFrame)) });
420
+ const assetProgress = createStageProgress(options.onProgress, "assets", tracks);
421
+ for (const [index, track] of tracks.entries()) {
422
+ assetProgress(index, 0);
423
+ await track.run((done) => assetProgress(index, done));
424
+ assetProgress(index, track.seconds);
313
425
  }
314
- runFfmpeg("The end card's backdrop", buildLastFrameArgs(edl, clipFiles, join(assetsDir, "last.jpg")));
426
+ const musicAsset = musicFile === null ? null : "assets/music.mp3";
315
427
  const fonts = copyFonts(options.brand.fonts, join(assetsDir, "fonts"));
316
428
  if (options.brand.logo !== null)
317
429
  cpSync(options.brand.logo, join(assetsDir, "logo.svg"));
@@ -323,7 +435,19 @@ export async function renderFootage(film, clips, transcript, edl, options) {
323
435
  sfx[event] = `assets/sfx/${name}`;
324
436
  }
325
437
  cpSync(require.resolve("gsap/dist/gsap.min.js"), join(assetsDir, "gsap.min.js"));
326
- const layout = resolveLayout(film.format, options.layout ?? {});
438
+ const base = resolveLayout(film.format, options.layout ?? {});
439
+ // A caption preset places the 9:16 box in a platform's safe zone; the project's own caption override still wins.
440
+ const preset = film.captionPreset === undefined || film.format !== "9:16" ? null : getCaptionPresetLayout(film.captionPreset, base.caption.fontSize);
441
+ const layout = preset === null || options.layout?.caption !== undefined ? base : { ...base, caption: preset };
442
+ const missing = findMissingCaptionGlyphs({
443
+ fonts: options.brand.fonts,
444
+ captions: film.captions ? words.map((word) => word.text) : [],
445
+ endCard: film.endCard,
446
+ brandName: options.brand.name,
447
+ });
448
+ if (missing.length > 0) {
449
+ return err(`the brand fonts cannot draw every letter of the film: ${missing.map((entry) => `${entry.font} lacks ${entry.characters.join(" ")}`).join("; ")}. Add a font file that covers them (brand.fonts).`);
450
+ }
327
451
  const html = composeFootage({
328
452
  film,
329
453
  edl,
@@ -347,12 +471,97 @@ export async function renderFootage(film, clips, transcript, edl, options) {
347
471
  writeFileSync(join(buildDir, "index.html"), html);
348
472
  mkdirSync(dirname(options.output), { recursive: true });
349
473
  log(`render: ${edl.segments.length} segments, ${seconds.toFixed(1)} s.`);
350
- const result = runHyperframes(["render", "--quality", options.quality, "-o", options.output], { cwd: buildDir, stdio: "inherit" });
351
- if (result.error !== undefined)
352
- return err(`hyperframes render did not start: ${result.error.message}.`);
353
- if (result.status !== 0)
354
- return err(`hyperframes render ended with code ${String(result.status)} (signal ${result.signal ?? "-"}).`);
355
- return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder) });
474
+ const renderFile = resolve(buildDir, "render.mp4");
475
+ const rendered = await runHyperframesRender(options, renderFile, seconds);
476
+ if (!rendered.ok)
477
+ return rendered;
478
+ const roles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.clip.role]));
479
+ const finished = await finishFootage(film, { renderFile, seconds, coverAt: film.cover?.at ?? getDefaultCoverSeconds(edl, roles) }, options);
480
+ if (!finished.ok)
481
+ return finished;
482
+ return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder), ...finished.value });
483
+ }
484
+ /**
485
+ * The final pass over hyperframes' render: publishable codecs and fast start (stream copy where the render already
486
+ * matches), checked with ffprobe, then the cover and the file's facts.
487
+ */
488
+ async function finishFootage(film, input, options) {
489
+ const { log, signal, output } = options;
490
+ const aborting = signal === undefined ? {} : { signal };
491
+ const progress = createStageProgress(options.onProgress, "finish", [{ item: "film", seconds: input.seconds }]);
492
+ progress(0, 0);
493
+ const rendered = await probeFilmFacts(input.renderFile, aborting);
494
+ if (!rendered.ok)
495
+ return err(`the rendered film: ${rendered.error}`);
496
+ const isCopied = isPublishableVideo(rendered.facts.video) && (rendered.facts.audio === null || isPublishableAudio(rendered.facts.audio));
497
+ await runFfmpeg("The film's final pass", buildFinishArgs(input.renderFile, rendered.facts, output), { ...aborting, output, onSeconds: (done) => progress(0, done) });
498
+ rmSync(input.renderFile, { force: true });
499
+ const finished = await probeFilmFacts(output, aborting);
500
+ if (!finished.ok)
501
+ return err(`the finished film: ${finished.error}`);
502
+ const problems = findFinishProblems(finished.facts, readTopLevelBoxes(output));
503
+ if (problems.length > 0)
504
+ return err(`the finished film ${output} is not publishable: ${problems.join("; ")}.`);
505
+ progress(0, input.seconds);
506
+ const { bytes, width, height, durationSeconds } = finished.facts;
507
+ log(`finish: ${isCopied ? "streams copied" : "re-encoded"} to H.264 High, AAC 48 kHz stereo, fast start; ${width}×${height}, ${durationSeconds.toFixed(2)} s, ${bytes} bytes.`);
508
+ let cover = null;
509
+ if (film.cover !== null) {
510
+ cover = getCoverPath(output);
511
+ const at = clampCoverSeconds(input.coverAt, durationSeconds);
512
+ if (at !== input.coverAt)
513
+ log(`cover: ${input.coverAt} s is past the film's end, the frame at ${at} s is used.`);
514
+ await runFfmpeg("The cover frame", buildCoverArgs(output, at, cover), { ...aborting, output: cover });
515
+ log(`cover: ${cover} (at ${at} s).`);
516
+ }
517
+ return ok({ bytes, width, height, durationSeconds, cover });
518
+ }
519
+ /** hyperframes' progress lines carry a percentage (`… 42%`); the last one on a line counts. */
520
+ export function readRenderPercent(line) {
521
+ const matches = [...line.matchAll(/(\d{1,3}(?:\.\d+)?)\s?%/g)];
522
+ const last = matches.at(-1)?.[1];
523
+ if (last === undefined)
524
+ return null;
525
+ const percent = Number(last);
526
+ return percent >= 0 && percent <= 100 ? percent : null;
527
+ }
528
+ /** The hyperframes render as a child process: its lines go to the log (progress every 10 %), its percentages to `onProgress`. */
529
+ async function runHyperframesRender(options, output, seconds) {
530
+ const progress = createStageProgress(options.onProgress, "render", [{ item: "film", seconds }]);
531
+ progress(0, 0);
532
+ let loggedTenth = -1;
533
+ const onLine = (line) => {
534
+ if (line.trim().length === 0)
535
+ return;
536
+ const percent = readRenderPercent(line);
537
+ if (percent === null) {
538
+ options.log(line.trim());
539
+ return;
540
+ }
541
+ progress(0, (percent / 100) * seconds);
542
+ // A progress line per frame would flood the log: one line per tenth of the render is kept.
543
+ const tenth = Math.floor(percent / 10);
544
+ if (tenth > loggedTenth) {
545
+ loggedTenth = tenth;
546
+ options.log(line.trim());
547
+ }
548
+ };
549
+ const result = await runProcess({
550
+ command: process.execPath,
551
+ args: [getHyperframesBin(), "render", "--quality", options.quality, "-o", output],
552
+ cwd: options.buildDir,
553
+ env: getHyperframesEnv(),
554
+ ...(options.signal === undefined ? {} : { signal: options.signal }),
555
+ removeOnAbort: [output],
556
+ onStdoutLine: onLine,
557
+ onStderrLine: onLine,
558
+ });
559
+ if (result.kind === "not-started")
560
+ return err(`hyperframes render did not start: ${result.error}.`);
561
+ if (result.code !== 0)
562
+ return err(`hyperframes render ended with code ${String(result.code)} (signal ${result.signal ?? "-"}).`);
563
+ progress(0, seconds);
564
+ return ok(null);
356
565
  }
357
566
  /**
358
567
  * The whole footage film from a `FootageFilm` object, no `marketing.json` needed: prepare, transcribe (or the dry
@@ -377,7 +586,7 @@ export async function renderFootageFilm(film, options) {
377
586
  options.log(`cut: ${edlPath} (${edl.segments.length} segments), as it is; softure-marketing footage --step=cut builds it again.`);
378
587
  }
379
588
  else {
380
- edl = cutFootage(film, clips.value, transcript.value.words, edlPath);
589
+ edl = cutFootage(film, clips.value, transcript.value.words, edlPath, options.keepSentences === undefined ? {} : { keepSentences: options.keepSentences });
381
590
  options.log(`cut: ${edl.segments.length} segments written to ${edlPath}.`);
382
591
  }
383
592
  return renderFootage(film, clips.value, transcript.value, edl, options);