@softure-ai/marketing-kit 0.16.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,25 +1,31 @@
1
- import { existsSync, mkdirSync, rmSync } from "node:fs";
1
+ import { existsSync, mkdirSync, readFileSync, rmSync } from "node:fs";
2
2
  import { join } from "node:path";
3
3
  import { setTimeout as sleep } from "node:timers/promises";
4
4
 
5
5
  import { describeProviderFailure, type ProviderFailure } from "../voice/provider-failure.js";
6
6
  import { err, ok } from "./failure.js";
7
7
  import { probeAudio } from "./audio.js";
8
- import { prepareClipSounds } from "./clip-sound.js";
8
+ import { prepareClipSounds, type ClipSound } from "./clip-sound.js";
9
+ import { isSilent, readLoudnessMeasurement } from "./loudness.js";
9
10
  import { writeCacheFile } from "./cache.js";
10
11
  import type { Edl } from "./edl.js";
11
12
  import type { FootageFilm } from "./film.js";
12
13
  import { getClipCacheKey } from "./paid.js";
13
14
  import { getFfmpegReporting, type FootageResult, type FootageRunOptions, type PreparedClip } from "./pipeline.js";
14
- import { runFfmpeg, withFfmpegFailureReporting } from "./process.js";
15
+ import { runFfmpeg, runProcess, withFfmpegFailureReporting } from "./process.js";
15
16
  import { toPaidAudio, type IsolationProvider, type PaidAudio } from "./providers.js";
16
17
  import {
17
18
  buildRangeAudioArgs,
18
19
  buildRangeLayArgs,
20
+ buildRangeMeasureArgs,
21
+ formatSpeechRangeGain,
19
22
  getPlanOptions,
23
+ getRangeInputGainDb,
24
+ getSpeechRangeGainPath,
20
25
  getSpeechRangeCache,
21
26
  getSpeechRangeWindows,
22
27
  planSpeechRanges,
28
+ readSpeechRangeGain,
23
29
  DEFAULT_SPEECH_RANGE_FADE_MS,
24
30
  type LaidRange,
25
31
  type PlanSpeechRangesOptions,
@@ -36,7 +42,9 @@ import {
36
42
  * (issue #1127): only a cancel stops the call. The ranges reach the provider's minimum input (`minSeconds`, issue
37
43
  * #1140): near ranges of a clip are merged, a short one is widened with the clip's own context and only the part the
38
44
  * cut hears is laid; a range that still cannot pass is never sent. A refusal the provider marks not retryable
39
- * (`isRetryable: false`, e.g. a 400 `audio_too_short`) is not asked again (issue #1141).
45
+ * (`isRetryable: false`, e.g. a 400 `audio_too_short`) is not asked again (issue #1141). Every window lands at the
46
+ * level of the clip's pre-gained sound (issue #1159): a bought range records the gain it was cut at next to its cached
47
+ * file, and a cached range cut from the un-gained sound (bought before the pre-gain) is raised by the clip's pre-gain.
40
48
  */
41
49
 
42
50
  /**
@@ -104,11 +112,18 @@ export interface IsolatedSpeechRange extends SpeechRange {
104
112
  * sent, its window keeps the clip's own sound); `not_bought` without `isCommit`.
105
113
  */
106
114
  outcome: SpeechRangeOutcome;
115
+ /**
116
+ * The dB its cleaned voice was raised by when laid (issue #1159): the clip's pre-gain for a range cut from the
117
+ * clip's un-gained sound, else 0; 0 when it is not laid.
118
+ */
119
+ gainDb: number;
107
120
  }
108
121
 
109
122
  export interface IsolatedSpeech {
110
123
  /** Every clip's audio for the speech track: its laid file, or its lossless sound when none of its ranges is laid. */
111
124
  speech: Record<string, string>;
125
+ /** Each clip's pre-gain in dB (`prepareClipSounds`): the gain of the sound its speech file is made from (issue #1159). */
126
+ speechGainDb: Record<string, number>;
112
127
  ranges: IsolatedSpeechRange[];
113
128
  }
114
129
 
@@ -155,11 +170,11 @@ async function runIsolateSpeechRanges(
155
170
  }
156
171
  const cache = options.rangeCache?.(range, clipKey) ?? getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider: providers.isolation.id });
157
172
  if (range.skip !== undefined) {
158
- ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached: false, laid: false, outcome: { kind: "skipped", ...range.skip } });
173
+ ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached: false, laid: false, outcome: { kind: "skipped", ...range.skip }, gainDb: 0 });
159
174
  continue;
160
175
  }
161
176
  const cached = existsSync(cache.path);
162
- ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached, laid: cached, outcome: cached ? { kind: "cached" } : { kind: "not_bought" } });
177
+ ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached, laid: cached, outcome: cached ? { kind: "cached" } : { kind: "not_bought" }, gainDb: 0 });
163
178
  }
164
179
  const skipped = ranges.filter((range) => range.outcome.kind === "skipped");
165
180
  if (skipped.length > 0) {
@@ -206,6 +221,8 @@ async function runIsolateSpeechRanges(
206
221
  }
207
222
  const paid = answer.paid;
208
223
  writeCacheFile(range.path, paid.audio);
224
+ // The gain of the sound it was cut from, so a later call lays it at the level of the clip's sound then (#1159).
225
+ writeCacheFile(getSpeechRangeGainPath(range.path), formatSpeechRangeGain(sounds[clip.clip.id]?.gainDb ?? 0));
209
226
  range.laid = true;
210
227
  range.outcome = { kind: "bought", attempts: answer.attempts };
211
228
  options.onCharge?.({
@@ -229,25 +246,63 @@ async function runIsolateSpeechRanges(
229
246
  log(`voice isolation DRY RUN: ${String(missing.length)} range${missing.length === 1 ? "" : "s"} (${seconds.toFixed(1)} s) not cached, laid with the clip's own sound. Run with --commit to pay.`);
230
247
  }
231
248
  const speech: Record<string, string> = {};
249
+ const speechGainDb: Record<string, number> = {};
232
250
  for (const clip of clips) {
233
251
  const id = clip.clip.id;
234
252
  speech[id] = getSound(clip);
253
+ const clipGainDb = sounds[id]?.gainDb ?? 0;
254
+ speechGainDb[id] = clipGainDb;
235
255
  const laidRanges = ranges.filter((range) => range.clip === id && range.laid);
236
256
  if (laidRanges.length === 0) continue;
237
257
  const laid: LaidRange[] = [];
238
258
  for (const window of getSpeechRangeWindows(laidRanges)) {
239
- const file = laidRanges.find((range) => range === window.range)?.path;
240
- if (file === undefined) continue;
259
+ const range = laidRanges.find((candidate) => candidate === window.range);
260
+ if (range === undefined) continue;
261
+ const file = range.path;
241
262
  const probed = await probeAudio(file, aborting);
242
- if (!probed.ok) return err(`${describeRange(window.range)}: the cleaned voice at ${file} does not read (${probed.error}); delete it and run again.`);
243
- laid.push({ window, file, isMono: probed.probe.channels === 1 });
263
+ if (!probed.ok) return err(`${describeRange(range)}: the cleaned voice at ${file} does not read (${probed.error}); delete it and run again.`);
264
+ const inputGainDb = await getCachedInputGainDb(range, { sound: sounds[id], file, ...aborting });
265
+ const gainDb = Math.round((clipGainDb - inputGainDb) * 100) / 100 || 0;
266
+ range.gainDb = gainDb;
267
+ if (gainDb !== 0) log(`voice isolation: ${describeRange(range)} was cut from the clip's sound at ${String(inputGainDb)} dB, laid over it at ${String(clipGainDb)} dB: raised by ${String(gainDb)} dB.`);
268
+ laid.push({ window, file, isMono: probed.probe.channels === 1, gainDb });
244
269
  }
245
270
  const output = join(buildDir, "audio", `${id}.ranges.wav`);
246
271
  mkdirSync(join(buildDir, "audio"), { recursive: true });
247
272
  await runFfmpeg(`Laying the cleaned voice over clip ${id}`, buildRangeLayArgs(getSound(clip), laid, output, { fadeMs: options.fadeMs ?? DEFAULT_SPEECH_RANGE_FADE_MS }), { ...aborting, output });
248
273
  speech[id] = output;
249
274
  }
250
- return ok({ speech, ranges });
275
+ return ok({ speech, speechGainDb, ranges });
276
+ }
277
+
278
+ /**
279
+ * The gain of the sound a laid range's cleaned voice was cut from (issue #1159): the clip's pre-gain when it was bought
280
+ * in this call; its record (`getSpeechRangeGainPath`) when there is one; else told from its loudness against the same
281
+ * range of the clip's pre-gained sound (`getRangeInputGainDb`), for a range cached before the record existed.
282
+ */
283
+ async function getCachedInputGainDb(range: IsolatedSpeechRange, input: { sound: ClipSound | undefined; file: string; signal?: AbortSignal }): Promise<number> {
284
+ const clipGainDb = input.sound?.gainDb ?? 0;
285
+ if (range.outcome.kind === "bought" || input.sound === undefined) return clipGainDb;
286
+ const record = getSpeechRangeGainPath(input.file);
287
+ if (existsSync(record)) {
288
+ const recorded = readSpeechRangeGain(readFileSync(record, "utf8"));
289
+ if (recorded !== null) return recorded;
290
+ }
291
+ if (clipGainDb === 0) return 0;
292
+ const cleanedLufs = await measureRangeLoudness(`the cleaned voice of ${describeRange(range)}`, buildRangeMeasureArgs(input.file), input.signal);
293
+ const baseLufs = await measureRangeLoudness(`${describeRange(range)} of the clip's sound`, buildRangeMeasureArgs(input.sound.file, range), input.signal);
294
+ return getRangeInputGainDb({ clipGainDb, cleanedLufs, baseLufs });
295
+ }
296
+
297
+ /** Integrated loudness in LUFS of a measuring pass (`buildRangeMeasureArgs`); null for silence. Throws naming `label` when ffmpeg fails. */
298
+ async function measureRangeLoudness(label: string, args: string[], signal: AbortSignal | undefined): Promise<number | null> {
299
+ const step = `Measuring ${label}`;
300
+ const measured = await runProcess({ command: "ffmpeg", args, ...(signal === undefined ? {} : { signal }) });
301
+ if (measured.kind === "not-started") throw new Error(`${step}: ffmpeg did not start (${measured.error}).`);
302
+ if (measured.code !== 0) throw new Error(`${step}: ffmpeg ended with code ${String(measured.code)}: ${measured.stderr.trim().split("\n").slice(-3).join(" | ")}.`);
303
+ const measurement = readLoudnessMeasurement(measured.stderr);
304
+ if (measurement === null) throw new Error(`${step}: ffmpeg printed no loudnorm measurement.`);
305
+ return isSilent(measurement) ? null : measurement.inputI;
251
306
  }
252
307
 
253
308
  type IsolationAnswer = { ok: true; paid: PaidAudio; attempts: number } | { ok: false; error: string; attempts: number; providerFailure: ProviderFailure | null };
@@ -167,6 +167,12 @@ export type TranscribeOutcome =
167
167
  words: Record<string, TimedWord[]>;
168
168
  /** The audio each clip's speech is cut from (the cleaned voice when isolated). */
169
169
  speech: Record<string, string>;
170
+ /**
171
+ * Each clip's pre-gain in dB (`prepareClipSounds`, issue #1159): the gain of the sound `speech` is made from. An
172
+ * app that lays its own cached ranges raises one cut from the clip's un-gained sound by it. Always answered;
173
+ * optional so an app's own `ready` outcome still compiles.
174
+ */
175
+ speechGainDb?: Record<string, number>;
170
176
  };
171
177
 
172
178
  export function getFrame(film: Pick<FootageFilm, "format">): Frame {
@@ -295,10 +301,10 @@ async function decodeIsolatedVoice(cached: string, clipId: string, buildDir: str
295
301
  return output;
296
302
  }
297
303
 
298
- /** Each clip's lossless, pre-gained sound (`prepareClipSounds`), by clip id. */
299
- async function getClipSoundFiles(clips: PreparedClip[], buildDir: string, signal: AbortSignal | undefined): Promise<Record<string, string>> {
300
- const sounds = await prepareClipSounds(clips, { buildDir, ...(signal === undefined ? {} : { signal }) });
301
- return Object.fromEntries(Object.entries(sounds).map(([id, sound]) => [id, sound.file]));
304
+ /** Each clip's lossless, pre-gained sound (`prepareClipSounds`) and its pre-gain in dB, by clip id. */
305
+ async function getClipSoundFiles(clips: PreparedClip[], buildDir: string, signal: AbortSignal | undefined): Promise<{ speech: Record<string, string>; speechGainDb: Record<string, number> }> {
306
+ const sounds = Object.entries(await prepareClipSounds(clips, { buildDir, ...(signal === undefined ? {} : { signal }) }));
307
+ return { speech: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.file])), speechGainDb: Object.fromEntries(sounds.map(([id, sound]) => [id, sound.gainDb])) };
302
308
  }
303
309
 
304
310
  /** The free transcripts of `--placeholder`, written under `placeholder/` for a look; the speech is each clip's lossless sound. */
@@ -308,7 +314,7 @@ async function transcribePlaceholder(clips: PreparedClip[], buildDir: string, si
308
314
  words[clip.clip.id] = clip.clip.role === "talk" ? makePlaceholderTranscript(clip.clip.script, clip.duration) : [];
309
315
  writeTranscript(join(buildDir, "placeholder", `${clip.clip.id}.transcript.json`), words[clip.clip.id] ?? []);
310
316
  }
311
- return { kind: "ready", words, speech: await getClipSoundFiles(clips, buildDir, signal) };
317
+ return { kind: "ready", words, ...(await getClipSoundFiles(clips, buildDir, signal)) };
312
318
  }
313
319
 
314
320
  /** The paid music step, when the film's music still has to be bought; null when it is cached or free. */
@@ -357,7 +363,7 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
357
363
  const words: Record<string, TimedWord[]> = {};
358
364
  for (const clip of clips) words[clip.clip.id] = [];
359
365
  // Every clip's sound, lossless and pre-gained: what the isolator, the speech to text and the speech track hear (#1139).
360
- const speech = await getClipSoundFiles(clips, buildDir, signal);
366
+ const { speech, speechGainDb } = await getClipSoundFiles(clips, buildDir, signal);
361
367
  const isolating = plan.filter(({ isolated }) => isolated !== null && !existsSync(isolated.path));
362
368
  const transcribing = plan.filter(({ transcript }) => !existsSync(transcript.path));
363
369
  const isolateProgress = createStageProgress(options.onProgress, "isolate", isolating.map(({ clip }) => ({ item: clip.clip.id, seconds: clip.duration })));
@@ -410,7 +416,7 @@ async function runTranscribeFootage(film: FootageFilm, clips: PreparedClip[], op
410
416
  transcribeProgress(index, clip.duration);
411
417
  words[id] = result.value.words;
412
418
  }
413
- return ok({ kind: "ready", words, speech });
419
+ return ok({ kind: "ready", words, speech, speechGainDb });
414
420
  }
415
421
 
416
422
  /** The talking clips' sentences with their ids (`s1`…), as `buildEdl` names them, for choosing what to keep. */
@@ -214,11 +214,62 @@ export function buildRangeAudioArgs(audio: string, range: Pick<SpeechRange, "sta
214
214
  return ["-v", "error", "-y", "-ss", seconds(range.startMs), "-t", seconds(range.endMs - range.startMs), "-i", audio, "-vn", ...codec, output];
215
215
  }
216
216
 
217
- /** A cleaned range laid back: its file, and whether it is mono (panned to both channels at full level). */
217
+ /**
218
+ * A cleaned range laid back: its file, whether it is mono (panned to both channels at full level), and the dB it is
219
+ * raised by before its fades (`gainDb`, issue #1159: a range cut from the clip's un-gained sound is raised by the
220
+ * clip's pre-gain; absent or 0 leaves it as it is).
221
+ */
218
222
  export interface LaidRange {
219
223
  window: SpeechRangeWindow;
220
224
  file: string;
221
225
  isMono: boolean;
226
+ gainDb?: number;
227
+ }
228
+
229
+ /**
230
+ * Where the gain of the sound a range's cleaned voice was cut from is recorded (issue #1159): next to its cached file,
231
+ * as `<path>.gain.json` holding `{ "inputGainDb": <dB> }`. Commit it with the cached file.
232
+ */
233
+ export function getSpeechRangeGainPath(path: string): string {
234
+ return `${path}.gain.json`;
235
+ }
236
+
237
+ /** The record of `getSpeechRangeGainPath`'s file. */
238
+ export function formatSpeechRangeGain(inputGainDb: number): string {
239
+ return `${JSON.stringify({ inputGainDb })}\n`;
240
+ }
241
+
242
+ /** The input gain in a record's text; null when the text is not such a record. */
243
+ export function readSpeechRangeGain(text: string): number | null {
244
+ let json: unknown;
245
+ try {
246
+ json = JSON.parse(text);
247
+ } catch {
248
+ return null;
249
+ }
250
+ if (typeof json !== "object" || json === null) return null;
251
+ const value = (json as Record<string, unknown>).inputGainDb;
252
+ return typeof value === "number" && Number.isFinite(value) ? value : null;
253
+ }
254
+
255
+ /** A measuring pass (`loudnorm`, nothing written) over `file`, or only over `range` of it, as 48 kHz stereo. */
256
+ export function buildRangeMeasureArgs(file: string, range?: Pick<SpeechRange, "startMs" | "endMs">): string[] {
257
+ const window = range === undefined ? [] : ["-ss", seconds(range.startMs), "-t", seconds(range.endMs - range.startMs)];
258
+ const format = `aformat=sample_rates=${String(FOOTAGE_SAMPLE_RATE)}:channel_layouts=stereo`;
259
+ return ["-hide_banner", "-nostats", ...window, "-i", file, "-map", "0:a:0", "-vn", "-af", `${format},loudnorm=I=-23:TP=-2:LRA=11:print_format=json`, "-f", "null", "-"];
260
+ }
261
+
262
+ /**
263
+ * The gain of the sound a cached cleaned range was cut from, told from its loudness (issue #1159): a range cut from
264
+ * the clip's un-gained sound sits about `clipGainDb` under the same range of the pre-gained sound, one cut from the
265
+ * pre-gained sound about level with it (the cleaning takes off a few dB at most). The nearer case wins: 0 (un-gained)
266
+ * or `clipGainDb`. Without a loudness on either side (null: silence) it is `clipGainDb`, so nothing is changed.
267
+ */
268
+ export function getRangeInputGainDb(options: { clipGainDb: number; cleanedLufs: number | null; baseLufs: number | null }): number {
269
+ const { clipGainDb, cleanedLufs, baseLufs } = options;
270
+ if (clipGainDb === 0 || cleanedLufs === null || baseLufs === null) return clipGainDb;
271
+ const difference = cleanedLufs - baseLufs;
272
+ return Math.abs(difference + clipGainDb) < Math.abs(difference) ? 0 : clipGainDb;
222
273
  }
223
274
 
224
275
  /** Milliseconds of the linear crossfade at every join between the clip's own sound and a cleaned window. */
@@ -281,17 +332,18 @@ export function buildRangeLayArgs(base: string, laid: readonly LaidRange[], outp
281
332
  const inputs = ["-i", base];
282
333
  const gains: string[] = [];
283
334
  const parts: string[] = [];
284
- laid.forEach(({ window, file, isMono }, index) => {
335
+ laid.forEach(({ window, file, isMono, gainDb }, index) => {
285
336
  const fade = byWindow.get(window);
286
337
  if (fade === undefined) return;
287
338
  inputs.push("-i", file);
288
339
  gains.push(buildGainExpression(fade));
289
340
  const channels = isMono ? "pan=stereo|c0=c0|c1=c0," : "";
341
+ const volume = gainDb === undefined || gainDb === 0 ? "" : `,volume=${String(gainDb)}dB`;
290
342
  const fadeIn = fade.inEndMs > fade.inStartMs ? `,afade=t=in:st=0:d=${seconds(fade.inEndMs - fade.inStartMs)}` : "";
291
343
  const fadeOut = fade.outEndMs > fade.outStartMs ? `,afade=t=out:st=${seconds(fade.outStartMs - fade.inStartMs)}:d=${seconds(fade.outEndMs - fade.outStartMs)}` : "";
292
344
  const delay = Math.round((fade.inStartMs * FOOTAGE_SAMPLE_RATE) / 1000);
293
345
  parts.push(
294
- `[${index + 1}:a]atrim=start=${seconds(fade.inStartMs - window.range.startMs)}:end=${seconds(fade.outEndMs - window.range.startMs)},asetpts=PTS-STARTPTS,${channels}${format}${fadeIn}${fadeOut},adelay=${String(delay)}S:all=1[w${String(index)}]`,
346
+ `[${index + 1}:a]atrim=start=${seconds(fade.inStartMs - window.range.startMs)}:end=${seconds(fade.outEndMs - window.range.startMs)},asetpts=PTS-STARTPTS,${channels}${format}${volume}${fadeIn}${fadeOut},adelay=${String(delay)}S:all=1[w${String(index)}]`,
295
347
  );
296
348
  });
297
349
  parts.unshift(`[0:a]${format},aeval='val(ch)*(1-(${gains.join("+")}))':c=same[base]`);
package/src/index.ts CHANGED
@@ -163,6 +163,7 @@ export {
163
163
  countOgSlides,
164
164
  getDefaultOgSize,
165
165
  OG_TEMPLATE_IDS,
166
+ PHOTO_COVER_LAYOUTS,
166
167
  getOgPalette,
167
168
  loadOgFonts,
168
169
  renderConfiguredOgImage,
@@ -175,6 +176,8 @@ export {
175
176
  type OgTemplateId,
176
177
  type PhotoCover,
177
178
  type PhotoCoverInput,
179
+ type PhotoCoverLayout,
180
+ type PhotoCoverTone,
178
181
  } from "./og/index.js";
179
182
  export { buildPosts, postsMarkdown, type PlatformPost, type PostsInput } from "./posts/posts.js";
180
183
  export { createActionScene, getLocator, runAction, type LocatorSource } from "./record/actions.js";
@@ -535,10 +538,15 @@ export {
535
538
  DEFAULT_SPEECH_RANGE_MIN_PAD,
536
539
  buildRangeAudioArgs,
537
540
  buildRangeLayArgs,
541
+ buildRangeMeasureArgs,
542
+ formatSpeechRangeGain,
543
+ getRangeInputGainDb,
538
544
  getSpeechRangeCache,
539
545
  getSpeechRangeFades,
546
+ getSpeechRangeGainPath,
540
547
  getSpeechRangeWindows,
541
548
  planSpeechRanges,
549
+ readSpeechRangeGain,
542
550
  type LaidRange,
543
551
  type PlanSpeechRangesOptions,
544
552
  type SpeechRange,
package/src/og/index.ts CHANGED
@@ -20,6 +20,9 @@ export {
20
20
  } from "./render.js";
21
21
  export {
22
22
  MAX_COVER_CAPTION_LENGTH,
23
+ MAX_COVER_KICKER_LENGTH,
24
+ MAX_COVER_PILL_LENGTH,
25
+ PHOTO_COVER_LAYOUTS,
23
26
  PHOTO_COVER_QUALITY,
24
27
  PHOTO_COVER_SIZE,
25
28
  getCoverBand,
@@ -28,6 +31,8 @@ export {
28
31
  type CoverBand,
29
32
  type PhotoCover,
30
33
  type PhotoCoverInput,
34
+ type PhotoCoverLayout,
35
+ type PhotoCoverTone,
31
36
  } from "./photo-cover.js";
32
37
  export {
33
38
  getCoverCrop,