@softure-ai/marketing-kit 0.14.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/CHANGELOG.md +95 -0
  2. package/README.md +136 -12
  3. package/dist/config/schema.d.ts +2 -1
  4. package/dist/config/schema.d.ts.map +1 -1
  5. package/dist/config/schema.js +7 -0
  6. package/dist/config/schema.js.map +1 -1
  7. package/dist/footage/clip-sound.d.ts +58 -0
  8. package/dist/footage/clip-sound.d.ts.map +1 -0
  9. package/dist/footage/clip-sound.js +103 -0
  10. package/dist/footage/clip-sound.js.map +1 -0
  11. package/dist/footage/declick.d.ts +1 -1
  12. package/dist/footage/declick.d.ts.map +1 -1
  13. package/dist/footage/declick.js +3 -4
  14. package/dist/footage/declick.js.map +1 -1
  15. package/dist/footage/draft.d.ts +19 -5
  16. package/dist/footage/draft.d.ts.map +1 -1
  17. package/dist/footage/draft.js +94 -24
  18. package/dist/footage/draft.js.map +1 -1
  19. package/dist/footage/elevenlabs.d.ts +5 -0
  20. package/dist/footage/elevenlabs.d.ts.map +1 -1
  21. package/dist/footage/elevenlabs.js +10 -3
  22. package/dist/footage/elevenlabs.js.map +1 -1
  23. package/dist/footage/failure.d.ts +53 -0
  24. package/dist/footage/failure.d.ts.map +1 -0
  25. package/dist/footage/failure.js +23 -0
  26. package/dist/footage/failure.js.map +1 -0
  27. package/dist/footage/fake.d.ts +10 -1
  28. package/dist/footage/fake.d.ts.map +1 -1
  29. package/dist/footage/fake.js +19 -4
  30. package/dist/footage/fake.js.map +1 -1
  31. package/dist/footage/film-mix.d.ts +5 -1
  32. package/dist/footage/film-mix.d.ts.map +1 -1
  33. package/dist/footage/film-mix.js +11 -2
  34. package/dist/footage/film-mix.js.map +1 -1
  35. package/dist/footage/hyperframes-capture.d.ts +30 -0
  36. package/dist/footage/hyperframes-capture.d.ts.map +1 -0
  37. package/dist/footage/hyperframes-capture.js +80 -0
  38. package/dist/footage/hyperframes-capture.js.map +1 -0
  39. package/dist/footage/isolate-ranges.d.ts +67 -5
  40. package/dist/footage/isolate-ranges.d.ts.map +1 -1
  41. package/dist/footage/isolate-ranges.js +92 -21
  42. package/dist/footage/isolate-ranges.js.map +1 -1
  43. package/dist/footage/loudness.d.ts +16 -2
  44. package/dist/footage/loudness.d.ts.map +1 -1
  45. package/dist/footage/loudness.js +10 -7
  46. package/dist/footage/loudness.js.map +1 -1
  47. package/dist/footage/paid.d.ts +14 -1
  48. package/dist/footage/paid.d.ts.map +1 -1
  49. package/dist/footage/paid.js +10 -4
  50. package/dist/footage/paid.js.map +1 -1
  51. package/dist/footage/pipeline.d.ts +10 -4
  52. package/dist/footage/pipeline.d.ts.map +1 -1
  53. package/dist/footage/pipeline.js +74 -64
  54. package/dist/footage/pipeline.js.map +1 -1
  55. package/dist/footage/providers.d.ts +5 -0
  56. package/dist/footage/providers.d.ts.map +1 -1
  57. package/dist/footage/providers.js.map +1 -1
  58. package/dist/footage/render.d.ts +1 -1
  59. package/dist/footage/render.d.ts.map +1 -1
  60. package/dist/footage/render.js +4 -3
  61. package/dist/footage/render.js.map +1 -1
  62. package/dist/footage/speech-ranges.d.ts +80 -14
  63. package/dist/footage/speech-ranges.d.ts.map +1 -1
  64. package/dist/footage/speech-ranges.js +146 -26
  65. package/dist/footage/speech-ranges.js.map +1 -1
  66. package/dist/index.d.ts +10 -6
  67. package/dist/index.d.ts.map +1 -1
  68. package/dist/index.js +9 -5
  69. package/dist/index.js.map +1 -1
  70. package/dist/og/index.d.ts +2 -0
  71. package/dist/og/index.d.ts.map +1 -1
  72. package/dist/og/index.js +2 -0
  73. package/dist/og/index.js.map +1 -1
  74. package/dist/og/photo-cover.d.ts +55 -0
  75. package/dist/og/photo-cover.d.ts.map +1 -0
  76. package/dist/og/photo-cover.js +215 -0
  77. package/dist/og/photo-cover.js.map +1 -0
  78. package/dist/og/photo.d.ts +51 -0
  79. package/dist/og/photo.d.ts.map +1 -0
  80. package/dist/og/photo.js +154 -0
  81. package/dist/og/photo.js.map +1 -0
  82. package/dist/voice/provider-failure.d.ts +41 -0
  83. package/dist/voice/provider-failure.d.ts.map +1 -0
  84. package/dist/voice/provider-failure.js +68 -0
  85. package/dist/voice/provider-failure.js.map +1 -0
  86. package/dist/voice/provider.d.ts +7 -1
  87. package/dist/voice/provider.d.ts.map +1 -1
  88. package/dist/voice/provider.js +1 -1
  89. package/dist/voice/provider.js.map +1 -1
  90. package/package.json +2 -1
  91. package/schema/marketing.schema.json +6 -0
  92. package/src/config/schema.ts +7 -0
  93. package/src/footage/clip-sound.ts +124 -0
  94. package/src/footage/declick.ts +3 -4
  95. package/src/footage/draft.ts +125 -24
  96. package/src/footage/elevenlabs.ts +11 -3
  97. package/src/footage/failure.ts +57 -0
  98. package/src/footage/fake.ts +28 -4
  99. package/src/footage/film-mix.ts +10 -2
  100. package/src/footage/hyperframes-capture.ts +100 -0
  101. package/src/footage/isolate-ranges.ts +158 -22
  102. package/src/footage/loudness.ts +17 -8
  103. package/src/footage/paid.ts +29 -4
  104. package/src/footage/pipeline.ts +85 -66
  105. package/src/footage/providers.ts +5 -0
  106. package/src/footage/render.ts +4 -3
  107. package/src/footage/speech-ranges.ts +197 -27
  108. package/src/index.ts +55 -1
  109. package/src/og/index.ts +23 -0
  110. package/src/og/photo-cover.ts +280 -0
  111. package/src/og/photo.ts +177 -0
  112. package/src/voice/provider-failure.ts +98 -0
  113. package/src/voice/provider.ts +7 -2
@@ -22,8 +22,14 @@ export interface FilmMixInput {
22
22
 
23
23
  const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
24
24
 
25
- /** The ffmpeg arguments that write the film's mix. */
25
+ /**
26
+ * The ffmpeg arguments that write the film's mix. The length is bounded three times (the padding's `whole_dur`, the
27
+ * trim and the output's `-t`): a mix that once ran unbounded on CI wrote a 90 GB WAV and filled the disk, so no single
28
+ * filter's end of stream is trusted with it. A film length that is not a positive number is a bug and throws.
29
+ */
26
30
  export function buildFilmMixArgs(input: FilmMixInput): string[] {
31
+ if (!Number.isFinite(input.seconds) || input.seconds <= 0) throw new Error(`buildFilmMixArgs: the film's length must be a positive number of seconds, got ${String(input.seconds)}.`);
32
+ const length = String(r3(input.seconds));
27
33
  const format = `aresample=${FOOTAGE_SAMPLE_RATE},aformat=sample_fmts=fltp:channel_layouts=stereo`;
28
34
  const sources: { file: string; filter: string }[] = [
29
35
  { file: input.speech.file, filter: `atrim=duration=${r3(input.speech.seconds)},${format}` },
@@ -36,7 +42,7 @@ export function buildFilmMixArgs(input: FilmMixInput): string[] {
36
42
  const chains = sources.map((source, index) => `[${index}:a]${source.filter}[a${index}]`);
37
43
  const labels = sources.map((_, index) => `[a${index}]`).join("");
38
44
  const mixed = sources.length === 1 ? `[a0]anull` : `${labels}amix=inputs=${sources.length}:duration=longest:normalize=0`;
39
- const graph = [...chains, `${mixed},apad,atrim=duration=${r3(input.seconds)}[mix]`].join(";");
45
+ const graph = [...chains, `${mixed},apad=whole_dur=${length},atrim=duration=${length}[mix]`].join(";");
40
46
  return [
41
47
  "-v",
42
48
  "error",
@@ -46,6 +52,8 @@ export function buildFilmMixArgs(input: FilmMixInput): string[] {
46
52
  graph,
47
53
  "-map",
48
54
  "[mix]",
55
+ "-t",
56
+ length,
49
57
  "-c:a",
50
58
  "pcm_f32le",
51
59
  "-ar",
@@ -0,0 +1,100 @@
1
+ /**
2
+ * How hyperframes captured a render, read from the summary it prints at the end (issue #1145). hyperframes 0.9.6 prints
3
+ * `beginframe capture · software gpu · compile 0.8s · extract 8.7s · … · assemble 0.6s` and, after a screenshot
4
+ * capture it did not ask for, a line `Screenshot capture (slower): BeginFrame did not run. …`.
5
+ */
6
+
7
+ /** hyperframes' capture modes: `beginframe` (fast), `screenshot` (slower), `drawelement`. */
8
+ export type HyperframesCapturePath = "beginframe" | "screenshot" | "drawelement";
9
+
10
+ /** The browser's GPU mode for the render. */
11
+ export type HyperframesGpuMode = "software" | "hardware";
12
+
13
+ /** The pipeline stages hyperframes times, in the order it prints them. */
14
+ export const HYPERFRAMES_CAPTURE_STAGES = ["compile", "extract", "audio", "probe", "setup", "capture", "encode", "assemble"] as const;
15
+
16
+ export type HyperframesCaptureStage = (typeof HYPERFRAMES_CAPTURE_STAGES)[number];
17
+
18
+ export interface HyperframesCapture {
19
+ /** The capture path; null when hyperframes printed none. */
20
+ path: HyperframesCapturePath | null;
21
+ /** The browser's GPU mode; null when hyperframes printed none. */
22
+ gpu: HyperframesGpuMode | null;
23
+ /** Seconds per stage, only the stages hyperframes printed. */
24
+ seconds: Partial<Record<HyperframesCaptureStage, number>>;
25
+ /** The encode ran during the capture (`encode (during capture)`), so its seconds overlap the capture's. */
26
+ isEncodeStreamed: boolean;
27
+ /** hyperframes' hint after a screenshot capture (`Screenshot capture (slower): …`); null when it printed none. */
28
+ fallbackHint: string | null;
29
+ }
30
+
31
+ const SEPARATOR = " · ";
32
+ const CAPTURE_PATHS: readonly string[] = ["beginframe", "screenshot", "drawelement"];
33
+ const GPU_MODES: readonly string[] = ["software", "hardware"];
34
+ const STAGES: readonly string[] = HYPERFRAMES_CAPTURE_STAGES;
35
+ const FALLBACK_HINT_PREFIX = "Screenshot capture";
36
+ // SGR colour codes (`\x1b[2m…\x1b[22m`) that hyperframes' dim style may add.
37
+ // eslint-disable-next-line no-control-regex -- the escape character is what the pattern matches
38
+ const ANSI_PATTERN = /\u001B\[[0-9;]*m/g;
39
+
40
+ /** hyperframes' `formatDuration`: `8.7s` under a minute, `1m 31.4s` from a minute on. */
41
+ function readDurationSeconds(text: string): number | null {
42
+ const match = /^(?:(\d+)m )?(\d+(?:\.\d+)?)s$/.exec(text);
43
+ if (match === null) return null;
44
+ const minutes = match[1] === undefined ? 0 : Number(match[1]);
45
+ return Math.round((minutes * 60 + Number(match[2])) * 1000) / 1000;
46
+ }
47
+
48
+ type SummaryPart =
49
+ | { kind: "path"; path: HyperframesCapturePath }
50
+ | { kind: "gpu"; gpu: HyperframesGpuMode }
51
+ | { kind: "stage"; stage: HyperframesCaptureStage; seconds: number; isStreamed: boolean };
52
+
53
+ function readSummaryPart(part: string): SummaryPart | null {
54
+ const path = /^(\w+) capture$/.exec(part)?.[1];
55
+ if (path !== undefined) return CAPTURE_PATHS.includes(path) ? { kind: "path", path: path as HyperframesCapturePath } : null;
56
+ const gpu = /^(\w+) gpu$/.exec(part)?.[1];
57
+ if (gpu !== undefined) return GPU_MODES.includes(gpu) ? { kind: "gpu", gpu: gpu as HyperframesGpuMode } : null;
58
+ const stage = /^(\w+)( \(during capture\))? (.+)$/.exec(part);
59
+ if (stage === null) return null;
60
+ const [, name = "", streamed, duration = ""] = stage;
61
+ const seconds = readDurationSeconds(duration);
62
+ if (!STAGES.includes(name) || seconds === null) return null;
63
+ return { kind: "stage", stage: name as HyperframesCaptureStage, seconds, isStreamed: streamed !== undefined };
64
+ }
65
+
66
+ /** A summary line: every ` · ` part is a capture path, a GPU mode or a timed stage. Null for any other line. */
67
+ function readSummaryLine(line: string): Omit<HyperframesCapture, "fallbackHint"> | null {
68
+ const parts = line.split(SEPARATOR).map(readSummaryPart);
69
+ if (parts.length === 0 || parts.some((part) => part === null)) return null;
70
+ const capture: Omit<HyperframesCapture, "fallbackHint"> = { path: null, gpu: null, seconds: {}, isEncodeStreamed: false };
71
+ for (const part of parts) {
72
+ if (part === null) continue;
73
+ if (part.kind === "path") capture.path = part.path;
74
+ else if (part.kind === "gpu") capture.gpu = part.gpu;
75
+ else {
76
+ capture.seconds[part.stage] = part.seconds;
77
+ if (part.stage === "encode" && part.isStreamed) capture.isEncodeStreamed = true;
78
+ }
79
+ }
80
+ return capture;
81
+ }
82
+
83
+ /**
84
+ * The capture report from hyperframes' output lines (stdout and stderr, in order): the last summary line, plus the
85
+ * screenshot hint when one was printed. Null when no line is a summary (an older or quiet hyperframes).
86
+ */
87
+ export function readHyperframesCapture(lines: readonly string[]): HyperframesCapture | null {
88
+ let summary: Omit<HyperframesCapture, "fallbackHint"> | null = null;
89
+ let fallbackHint: string | null = null;
90
+ for (const raw of lines) {
91
+ const line = raw.replace(ANSI_PATTERN, "").trim();
92
+ if (line.length === 0) continue;
93
+ if (line.startsWith(FALLBACK_HINT_PREFIX)) {
94
+ fallbackHint = line;
95
+ continue;
96
+ }
97
+ summary = readSummaryLine(line) ?? summary;
98
+ }
99
+ return summary === null ? null : { ...summary, fallbackHint };
100
+ }
@@ -1,38 +1,95 @@
1
- import { existsSync, mkdirSync } from "node:fs";
1
+ import { existsSync, mkdirSync, rmSync } from "node:fs";
2
2
  import { join } from "node:path";
3
+ import { setTimeout as sleep } from "node:timers/promises";
3
4
 
4
- import { err, ok } from "../voice/provider.js";
5
+ import { describeProviderFailure, type ProviderFailure } from "../voice/provider-failure.js";
6
+ import { err, ok } from "./failure.js";
5
7
  import { probeAudio } from "./audio.js";
8
+ import { prepareClipSounds } from "./clip-sound.js";
6
9
  import { writeCacheFile } from "./cache.js";
7
10
  import type { Edl } from "./edl.js";
8
11
  import type { FootageFilm } from "./film.js";
9
12
  import { getClipCacheKey } from "./paid.js";
10
13
  import { getFfmpegReporting, type FootageResult, type FootageRunOptions, type PreparedClip } from "./pipeline.js";
11
14
  import { runFfmpeg, withFfmpegFailureReporting } from "./process.js";
12
- import { toPaidAudio } from "./providers.js";
15
+ import { toPaidAudio, type IsolationProvider, type PaidAudio } from "./providers.js";
13
16
  import {
14
17
  buildRangeAudioArgs,
15
18
  buildRangeLayArgs,
19
+ getPlanOptions,
16
20
  getSpeechRangeCache,
17
21
  getSpeechRangeWindows,
18
22
  planSpeechRanges,
23
+ DEFAULT_SPEECH_RANGE_FADE_MS,
19
24
  type LaidRange,
25
+ type PlanSpeechRangesOptions,
20
26
  type SpeechRange,
27
+ type SpeechRangeCacheResolver,
21
28
  } from "./speech-ranges.js";
22
29
 
23
30
  /**
24
31
  * Voice isolation of only what the cut hears (issue #1087): each talking clip's sound ranges (`planSpeechRanges`) are
25
32
  * cleaned one by one and cached by the clip's identity, the range in milliseconds and the provider, so an unchanged
26
33
  * segment is never paid twice and a changed cut pays only its new ranges. The cleaned ranges are laid back over the
27
- * clip's own sound into one file as long as the clip: the speech map `renderFootage` takes.
34
+ * clip's own sound into one file as long as the clip: the speech map `renderFootage` takes. A range the provider refuses
35
+ * is asked again, and one refused on every attempt keeps the clip's own sound in its window while the others are laid
36
+ * (issue #1127): only a cancel stops the call. The ranges reach the provider's minimum input (`minSeconds`, issue
37
+ * #1140): near ranges of a clip are merged, a short one is widened with the clip's own context and only the part the
38
+ * cut hears is laid; a range that still cannot pass is never sent. A refusal the provider marks not retryable
39
+ * (`isRetryable: false`, e.g. a 400 `audio_too_short`) is not asked again (issue #1141).
28
40
  */
29
41
 
42
+ /**
43
+ * How often a refused range is asked again: `attempts` in all (default 3), waiting `delaysMs[n]` before attempt n + 2.
44
+ * A failure whose `providerFailure.isRetryable` is false ends the range at once.
45
+ */
46
+ export interface SpeechRangeRetry {
47
+ /** Attempts per range, at least 1; default `DEFAULT_SPEECH_RANGE_ATTEMPTS` (3). */
48
+ attempts?: number;
49
+ /** Milliseconds before each retry (the last repeats); default `DEFAULT_SPEECH_RANGE_RETRY_DELAYS_MS` (1 s, 2 s). */
50
+ delaysMs?: readonly number[];
51
+ /** Waits `ms`, rejecting when `signal` aborts; default a timer. Injected by tests. */
52
+ wait?: (ms: number, signal?: AbortSignal) => Promise<void>;
53
+ }
54
+
55
+ export const DEFAULT_SPEECH_RANGE_ATTEMPTS = 3;
56
+ export const DEFAULT_SPEECH_RANGE_RETRY_DELAYS_MS: readonly number[] = [1000, 2000];
57
+
30
58
  export type IsolateSpeechRangesOptions = Pick<FootageRunOptions, "buildDir" | "cacheDir" | "providers" | "isCommit" | "signal" | "onCharge" | "onProgress" | "onFfmpegFailure"> & {
31
59
  /** Seconds added on each side of a sound span; default `DEFAULT_SPEECH_RANGE_MARGIN` (0.25). */
32
60
  margin?: number;
61
+ /** Ranges of a clip at most this many seconds apart are isolated as one; default 2, `null` keeps them apart. */
62
+ mergeGap?: PlanSpeechRangesOptions["mergeGap"];
63
+ /** Seconds a range shorter than the provider's minimum is widened past it; default `DEFAULT_SPEECH_RANGE_MIN_PAD` (0.4). */
64
+ minPad?: number;
65
+ /** Retries of a refused range. */
66
+ retry?: SpeechRangeRetry;
67
+ /** Milliseconds of the linear crossfade at every join; default `DEFAULT_SPEECH_RANGE_FADE_MS` (15), 0 switches at once. */
68
+ fadeMs?: number;
69
+ /** The `item` of a range's charge (e.g. the app's segment id); default the clip id. */
70
+ chargeItem?: (range: SpeechRange) => string;
71
+ /**
72
+ * Where a range's cleaned voice is cached; default `getSpeechRangeCache` (`<cacheDir>/<film>/<key>.isolated.mp3`).
73
+ * An app that already holds paid ranges under its own keys points the block at them.
74
+ */
75
+ rangeCache?: SpeechRangeCacheResolver;
33
76
  log?: (line: string) => void;
34
77
  };
35
78
 
79
+ /** What happened to a range in this call. */
80
+ export type SpeechRangeOutcome =
81
+ | { kind: "cached" }
82
+ | { kind: "bought"; attempts: number }
83
+ | {
84
+ kind: "failed";
85
+ error: string;
86
+ attempts: number;
87
+ /** The provider's structured failure of the last attempt (safe to log); null when it gave none. */
88
+ providerFailure: ProviderFailure | null;
89
+ }
90
+ | { kind: "skipped"; reason: "too_short"; minSeconds: number }
91
+ | { kind: "not_bought" };
92
+
36
93
  export interface IsolatedSpeechRange extends SpeechRange {
37
94
  /** The paid cache's key and file of the range's cleaned voice. */
38
95
  cacheKey: string;
@@ -41,10 +98,16 @@ export interface IsolatedSpeechRange extends SpeechRange {
41
98
  cached: boolean;
42
99
  /** Whether it is laid in the clip's speech file (cached, or bought with `isCommit`). */
43
100
  laid: boolean;
101
+ /**
102
+ * `cached`; `bought` in this call; `failed` when the provider refused it on every attempt (its window keeps the
103
+ * clip's own sound); `skipped` when it is shorter than the provider's minimum even after merging and widening (never
104
+ * sent, its window keeps the clip's own sound); `not_bought` without `isCommit`.
105
+ */
106
+ outcome: SpeechRangeOutcome;
44
107
  }
45
108
 
46
109
  export interface IsolatedSpeech {
47
- /** Every clip's audio for the speech track: its laid file, or the normalized clip when none of its ranges is laid. */
110
+ /** Every clip's audio for the speech track: its laid file, or its lossless sound when none of its ranges is laid. */
48
111
  speech: Record<string, string>;
49
112
  ranges: IsolatedSpeechRange[];
50
113
  }
@@ -52,8 +115,10 @@ export interface IsolatedSpeech {
52
115
  /**
53
116
  * Cleans the voice of the ranges the cut hears and answers the speech map the render takes (`TranscribeOutcome.speech`).
54
117
  * With `isCommit` every uncached range is bought (`onCharge` step `isolate`, item the clip id, with its `range`);
55
- * without it only cached ranges are laid, a free draft, and `ranges` names what is still missing. Each clip with a laid
56
- * range gets `<buildDir>/audio/<id>.ranges.mp3`.
118
+ * without it only cached ranges are laid, a free draft, and `ranges` names what is still missing. A refused range is
119
+ * retried (`retry`); one refused on every attempt is answered `failed` and the call still succeeds. Each clip with a
120
+ * laid range gets `<buildDir>/audio/<id>.ranges.wav`, crossfaded at every join (`fadeMs`). The isolator hears, and the
121
+ * ranges are laid over, each clip's lossless, pre-gained sound (`prepareClipSounds`, issue #1139).
57
122
  */
58
123
  export function isolateSpeechRanges(film: FootageFilm, clips: PreparedClip[], edl: Edl, options: IsolateSpeechRangesOptions): Promise<FootageResult<IsolatedSpeech>> {
59
124
  const log = options.log ?? ((): void => undefined);
@@ -75,7 +140,7 @@ async function runIsolateSpeechRanges(
75
140
  const planned = planSpeechRanges(
76
141
  edl,
77
142
  clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration })),
78
- options.margin === undefined ? {} : { margin: options.margin },
143
+ getPlanOptions(options, providers.isolation),
79
144
  );
80
145
  const byClip = new Map(clips.map((clip) => [clip.clip.id, clip]));
81
146
  const clipKeys = new Map<string, string>();
@@ -88,16 +153,28 @@ async function runIsolateSpeechRanges(
88
153
  clipKey = await getClipCacheKey(clip.source, clip.clip);
89
154
  clipKeys.set(range.clip, clipKey);
90
155
  }
91
- const cache = getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider: providers.isolation.id });
156
+ const cache = options.rangeCache?.(range, clipKey) ?? getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider: providers.isolation.id });
157
+ if (range.skip !== undefined) {
158
+ ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached: false, laid: false, outcome: { kind: "skipped", ...range.skip } });
159
+ continue;
160
+ }
92
161
  const cached = existsSync(cache.path);
93
- ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached, laid: cached });
162
+ ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached, laid: cached, outcome: cached ? { kind: "cached" } : { kind: "not_bought" } });
163
+ }
164
+ const skipped = ranges.filter((range) => range.outcome.kind === "skipped");
165
+ if (skipped.length > 0) {
166
+ const minimum = providers.isolation.minSeconds ?? 0;
167
+ log(`voice isolation: ${String(skipped.length)} range${skipped.length === 1 ? "" : "s"} shorter than ${providers.isolation.id}'s minimum of ${String(minimum)} s even with the clip's whole sound, not sent, laid with the clip's own sound: ${skipped.map(describeRange).join("; ")}.`);
94
168
  }
95
- const buying = options.isCommit ? ranges.filter((range) => !range.cached) : [];
169
+ const buying = options.isCommit ? ranges.filter((range) => range.outcome.kind === "not_bought") : [];
96
170
  const total = buying.reduce((sum, range) => sum + (range.endMs - range.startMs) / 1000, 0);
97
171
  let done = 0;
98
172
  const report = (range: SpeechRange, index: number): void => {
99
173
  options.onProgress?.({ step: "isolate", item: range.clip, itemIndex: index + 1, itemCount: buying.length, done: Math.round(done * 1000) / 1000, total: Math.round(total * 1000) / 1000 });
100
174
  };
175
+ // Every clip's sound, lossless and pre-gained (#1139): what the isolator hears and what the cleaned ranges are laid over.
176
+ const sounds = await prepareClipSounds(clips, { buildDir, ...aborting });
177
+ const getSound = (clip: PreparedClip): string => sounds[clip.clip.id]?.file ?? clip.file;
101
178
  const workDir = join(buildDir, "audio", "ranges");
102
179
  for (const [index, range] of buying.entries()) {
103
180
  const clip = byClip.get(range.clip);
@@ -106,19 +183,34 @@ async function runIsolateSpeechRanges(
106
183
  report(range, index);
107
184
  mkdirSync(workDir, { recursive: true });
108
185
  // The provider sees the clip's id and the range as the file name.
109
- const input = join(workDir, `${range.clip}.${String(range.startMs)}-${String(range.endMs)}.mp3`);
110
- await runFfmpeg(`Cutting ${describeRange(range)} for voice isolation`, buildRangeAudioArgs(clip.file, range, input), { ...aborting, output: input });
186
+ const input = join(workDir, `${range.clip}.${String(range.startMs)}-${String(range.endMs)}.wav`);
187
+ await runFfmpeg(`Cutting ${describeRange(range)} for voice isolation`, buildRangeAudioArgs(getSound(clip), range, input), { ...aborting, output: input });
111
188
  signal?.throwIfAborted();
112
189
  log(`voice isolation (${providers.isolation.id}): ${describeRange(range)}, ${units.toFixed(1)} s.`);
113
- const cleaned = await providers.isolation.isolate({ audio: input, ...aborting });
114
- signal?.throwIfAborted();
115
- if (!cleaned.ok) return err(`${describeRange(range)}: ${cleaned.error}`);
116
- const paid = toPaidAudio(cleaned.value);
190
+ const answer = await isolateWithRetries({
191
+ audio: input,
192
+ isolation: providers.isolation,
193
+ retry: options.retry ?? {},
194
+ ...aborting,
195
+ onRefused: (error, attempt, providerFailure) => {
196
+ const reason = providerFailure === null ? error : describeProviderFailure(providerFailure);
197
+ log(`voice isolation: ${describeRange(range)} refused (attempt ${String(attempt)}): ${reason}`);
198
+ },
199
+ });
200
+ if (!answer.ok) {
201
+ rmSync(input, { force: true });
202
+ range.outcome = { kind: "failed", error: answer.error, attempts: answer.attempts, providerFailure: answer.providerFailure };
203
+ done += units;
204
+ report(range, index);
205
+ continue;
206
+ }
207
+ const paid = answer.paid;
117
208
  writeCacheFile(range.path, paid.audio);
118
209
  range.laid = true;
210
+ range.outcome = { kind: "bought", attempts: answer.attempts };
119
211
  options.onCharge?.({
120
212
  step: "isolate",
121
- item: range.clip,
213
+ item: options.chargeItem?.(range) ?? range.clip,
122
214
  provider: providers.isolation.id,
123
215
  units,
124
216
  ...(paid.credits === undefined ? {} : { credits: paid.credits }),
@@ -129,7 +221,9 @@ async function runIsolateSpeechRanges(
129
221
  done += units;
130
222
  report(range, index);
131
223
  }
132
- const missing = ranges.filter((range) => !range.laid);
224
+ const failed = ranges.filter((range) => range.outcome.kind === "failed");
225
+ if (failed.length > 0) log(`voice isolation: ${String(failed.length)} range${failed.length === 1 ? "" : "s"} refused, laid with the clip's own sound: ${failed.map(describeRange).join("; ")}.`);
226
+ const missing = ranges.filter((range) => range.outcome.kind === "not_bought");
133
227
  if (missing.length > 0) {
134
228
  const seconds = missing.reduce((sum, range) => sum + (range.endMs - range.startMs) / 1000, 0);
135
229
  log(`voice isolation DRY RUN: ${String(missing.length)} range${missing.length === 1 ? "" : "s"} (${seconds.toFixed(1)} s) not cached, laid with the clip's own sound. Run with --commit to pay.`);
@@ -137,7 +231,7 @@ async function runIsolateSpeechRanges(
137
231
  const speech: Record<string, string> = {};
138
232
  for (const clip of clips) {
139
233
  const id = clip.clip.id;
140
- speech[id] = clip.file;
234
+ speech[id] = getSound(clip);
141
235
  const laidRanges = ranges.filter((range) => range.clip === id && range.laid);
142
236
  if (laidRanges.length === 0) continue;
143
237
  const laid: LaidRange[] = [];
@@ -148,14 +242,56 @@ async function runIsolateSpeechRanges(
148
242
  if (!probed.ok) return err(`${describeRange(window.range)}: the cleaned voice at ${file} does not read (${probed.error}); delete it and run again.`);
149
243
  laid.push({ window, file, isMono: probed.probe.channels === 1 });
150
244
  }
151
- const output = join(buildDir, "audio", `${id}.ranges.mp3`);
245
+ const output = join(buildDir, "audio", `${id}.ranges.wav`);
152
246
  mkdirSync(join(buildDir, "audio"), { recursive: true });
153
- await runFfmpeg(`Laying the cleaned voice over clip ${id}`, buildRangeLayArgs(clip.file, laid, output), { ...aborting, output });
247
+ await runFfmpeg(`Laying the cleaned voice over clip ${id}`, buildRangeLayArgs(getSound(clip), laid, output, { fadeMs: options.fadeMs ?? DEFAULT_SPEECH_RANGE_FADE_MS }), { ...aborting, output });
154
248
  speech[id] = output;
155
249
  }
156
250
  return ok({ speech, ranges });
157
251
  }
158
252
 
253
+ type IsolationAnswer = { ok: true; paid: PaidAudio; attempts: number } | { ok: false; error: string; attempts: number; providerFailure: ProviderFailure | null };
254
+
255
+ /** Asks the provider for one range's cleaned voice, again after each retryable refusal up to `retry.attempts`; a cancel throws. */
256
+ async function isolateWithRetries(input: {
257
+ audio: string;
258
+ isolation: IsolationProvider;
259
+ retry: SpeechRangeRetry;
260
+ signal?: AbortSignal;
261
+ onRefused: (error: string, attempt: number, providerFailure: ProviderFailure | null) => void;
262
+ }): Promise<IsolationAnswer> {
263
+ const { audio, isolation, retry, signal, onRefused } = input;
264
+ const attempts = Math.max(1, Math.floor(retry.attempts ?? DEFAULT_SPEECH_RANGE_ATTEMPTS));
265
+ const delays = retry.delaysMs ?? DEFAULT_SPEECH_RANGE_RETRY_DELAYS_MS;
266
+ const wait = retry.wait ?? waitFor;
267
+ let error = "";
268
+ let providerFailure: ProviderFailure | null = null;
269
+ for (let attempt = 1; attempt <= attempts; attempt += 1) {
270
+ if (attempt > 1) {
271
+ await wait(delays[Math.min(attempt - 2, delays.length - 1)] ?? 0, signal);
272
+ signal?.throwIfAborted();
273
+ }
274
+ const cleaned = await isolation.isolate({ audio, ...(signal === undefined ? {} : { signal }) });
275
+ signal?.throwIfAborted();
276
+ if (cleaned.ok) return { ok: true, paid: toPaidAudio(cleaned.value), attempts: attempt };
277
+ error = cleaned.error;
278
+ providerFailure = cleaned.providerFailure ?? null;
279
+ onRefused(error, attempt, providerFailure);
280
+ if (providerFailure?.isRetryable === false) return { ok: false, error, attempts: attempt, providerFailure };
281
+ }
282
+ return { ok: false, error, attempts, providerFailure };
283
+ }
284
+
285
+ /** Resolves after `ms`, or rejects with the signal's reason when it aborts first. */
286
+ async function waitFor(ms: number, signal?: AbortSignal): Promise<void> {
287
+ try {
288
+ await sleep(ms, undefined, signal === undefined ? {} : { signal });
289
+ } catch (error) {
290
+ signal?.throwIfAborted();
291
+ throw error;
292
+ }
293
+ }
294
+
159
295
  function describeRange(range: SpeechRange): string {
160
296
  return `clip ${range.clip} ${(range.startMs / 1000).toFixed(3)}-${(range.endMs / 1000).toFixed(3)} s`;
161
297
  }
@@ -2,6 +2,7 @@ import { copyFileSync } from "node:fs";
2
2
 
3
3
  import { z } from "zod";
4
4
 
5
+ import { getAudioCodecArgs } from "./clip-sound.js";
5
6
  import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
6
7
  import { runFfmpeg, runProcess } from "./process.js";
7
8
  import { FINISHED_AUDIO_BITRATE } from "./publish.js";
@@ -111,7 +112,7 @@ export function getLoudnessFilter(measurement: LoudnessMeasurement, target: Loud
111
112
  return `${filter},aresample=${FOOTAGE_SAMPLE_RATE}`;
112
113
  }
113
114
 
114
- /** The speech track's second pass, written as an mp3 at the film's rate. */
115
+ /** The speech track's second pass at the film's rate, encoded by the output's extension (`.wav` lossless float). */
115
116
  export function buildLoudnessArgs(options: { input: string; measurement: LoudnessMeasurement; target: LoudnessTarget; output: string }): string[] {
116
117
  return [
117
118
  "-v",
@@ -123,10 +124,7 @@ export function buildLoudnessArgs(options: { input: string; measurement: Loudnes
123
124
  getLoudnessFilter(options.measurement, options.target),
124
125
  "-ar",
125
126
  String(FOOTAGE_SAMPLE_RATE),
126
- "-c:a",
127
- "libmp3lame",
128
- "-q:a",
129
- "2",
127
+ ...getAudioCodecArgs(options.output),
130
128
  options.output,
131
129
  ];
132
130
  }
@@ -196,8 +194,16 @@ export interface MixLoudnessTarget extends LoudnessTarget {
196
194
  * `target ± warnTolerance`, under the true peak limit, finishes with a warning instead of failing. Absent: none.
197
195
  */
198
196
  warnTolerance?: number;
197
+ /**
198
+ * The finished sound's AAC bitrate in kbps (192 to 512), the film's only lossy audio encode (issue #1139). Absent:
199
+ * `FINISHED_AUDIO_BITRATE` (192).
200
+ */
201
+ audioBitrateKbps?: number;
199
202
  }
200
203
 
204
+ /** The lowest and highest AAC bitrate `MixLoudnessTarget.audioBitrateKbps` takes, in kbps. */
205
+ export const AUDIO_BITRATE_RANGE_KBPS = { min: 192, max: 512 } as const;
206
+
201
207
  /** A gain to the window's edge aims this far inside it (at most half the tolerance), so the encode's drift stays in. */
202
208
  const MIX_EDGE_MARGIN = 0.2;
203
209
  /** The limiter's ceiling sits this far under the true peak limit: AAC adds inter-sample overshoot after it. */
@@ -367,8 +373,11 @@ export function getMixFilter(attempt: MixAttempt): string {
367
373
  return [`volume=${attempt.gainDb}dB`, `aresample=${LIMITER_RATE}`, `alimiter=limit=${limit}:level=false:latency=true`, `aresample=${FOOTAGE_SAMPLE_RATE}`, `lowpass=f=${MIX_LOWPASS_HZ}`].join(",");
368
374
  }
369
375
 
370
- /** The mix's sound on its own through `filter`, encoded as the finished film carries it (AAC 192k cut at `MIX_LOWPASS_HZ`, 48 kHz, stereo). */
371
- export function buildMixArgs(options: { input: string; filter: string; output: string }): string[] {
376
+ /**
377
+ * The mix's sound on its own through `filter`, encoded as the finished film carries it (AAC at `bitrateKbps`, default
378
+ * `FINISHED_AUDIO_BITRATE`, cut at `MIX_LOWPASS_HZ`, 48 kHz, stereo).
379
+ */
380
+ export function buildMixArgs(options: { input: string; filter: string; output: string; bitrateKbps?: number }): string[] {
372
381
  return [
373
382
  "-v",
374
383
  "error",
@@ -383,7 +392,7 @@ export function buildMixArgs(options: { input: string; filter: string; output: s
383
392
  "-c:a",
384
393
  "aac",
385
394
  "-b:a",
386
- FINISHED_AUDIO_BITRATE,
395
+ options.bitrateKbps === undefined ? FINISHED_AUDIO_BITRATE : `${String(options.bitrateKbps)}k`,
387
396
  "-cutoff",
388
397
  String(MIX_LOWPASS_HZ),
389
398
  "-ar",
@@ -10,7 +10,15 @@ import { getFilmKeyTerms } from "./keyterms.js";
10
10
  import type { FootageClip, FootageFilm } from "./film.js";
11
11
  import { getTrimmedDuration, probeFootage } from "./probe.js";
12
12
  import type { FootageProviders, FootageRates, PaidStepKind } from "./providers.js";
13
- import { getSpeechRangeCache, planSpeechRanges, type SpeechRangeClip } from "./speech-ranges.js";
13
+ import {
14
+ getPlanOptions,
15
+ getSpeechRangeCache,
16
+ planSpeechRanges,
17
+ type PlanSpeechRangesOptions,
18
+ type SpeechRangeCacheResolver,
19
+ type SpeechRangeClip,
20
+ type SpeechRangeSkip,
21
+ } from "./speech-ranges.js";
14
22
 
15
23
  /**
16
24
  * Where each paid result of a footage film is cached, shared by the pipeline and by the questions an app asks before
@@ -161,6 +169,11 @@ export interface FootageEstimateItem {
161
169
  cached: boolean;
162
170
  /** The range of the clip an `isolate: "segments"` item prices, in seconds of the trimmed clip; absent: the whole clip. */
163
171
  range?: { start: number; end: number };
172
+ /**
173
+ * Set on an `isolate: "segments"` range that will not be sent: shorter than the provider's minimum even after
174
+ * merging and widening (`units` and `credits` 0).
175
+ */
176
+ skipped?: SpeechRangeSkip;
164
177
  }
165
178
 
166
179
  export interface FootageEstimate {
@@ -179,6 +192,12 @@ export interface EstimateFootageOptions extends FootageCacheOptions {
179
192
  isolate?: "clips" | "segments";
180
193
  /** Seconds around each sound span with `isolate: "segments"`; default `DEFAULT_SPEECH_RANGE_MARGIN`. */
181
194
  margin?: number;
195
+ /** Ranges of a clip at most this many seconds apart are priced as one with `isolate: "segments"`, as `isolateSpeechRanges` merges them. */
196
+ mergeGap?: PlanSpeechRangesOptions["mergeGap"];
197
+ /** Seconds a range shorter than the provider's minimum is widened past it with `isolate: "segments"`. */
198
+ minPad?: number;
199
+ /** Where a range's cleaned voice is cached with `isolate: "segments"`, as `isolateSpeechRanges` takes it; default `getSpeechRangeCache`. */
200
+ rangeCache?: SpeechRangeCacheResolver;
182
201
  }
183
202
 
184
203
  const round = (value: number, digits: number): number => {
@@ -232,9 +251,14 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
232
251
  }
233
252
  if (isSegments && edl !== null) {
234
253
  const provider = options.providers.isolation.id;
235
- for (const range of planSpeechRanges(edl, rangeClips, options.margin === undefined ? {} : { margin: options.margin })) {
254
+ for (const range of planSpeechRanges(edl, rangeClips, getPlanOptions(options, options.providers.isolation))) {
255
+ if (range.skip !== undefined) {
256
+ items.push({ step: "isolate", item: range.clip, provider, units: 0, unit: "seconds", credits: 0, cached: false, range: { start: range.startMs / 1000, end: range.endMs / 1000 }, skipped: range.skip });
257
+ continue;
258
+ }
236
259
  const clipKey = clipKeys[range.clip] ?? null;
237
- const cached = clipKey !== null && existsSync(getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider }).path);
260
+ const cache = clipKey === null ? null : (options.rangeCache?.(range, clipKey) ?? getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider }));
261
+ const cached = cache !== null && existsSync(cache.path);
238
262
  add("isolate", range.clip, provider, (range.endMs - range.startMs) / 1000, cached);
239
263
  const item = items.at(-1);
240
264
  if (item !== undefined) item.range = { start: range.startMs / 1000, end: range.endMs / 1000 };
@@ -250,7 +274,8 @@ export async function estimateFootageFilm(film: FootageFilm, options: EstimateFo
250
274
 
251
275
  /**
252
276
  * What isolating only the cut's ranges will cost (`isolateSpeechRanges`), for a confirmation step: the `isolate` items
253
- * of `estimateFootageFilm` with `isolate: "segments"`, one per range, a cached range at 0.
277
+ * of `estimateFootageFilm` with `isolate: "segments"`, one per range as it will be sent, a cached range at 0, a range
278
+ * shorter than the provider's minimum marked `skipped` at 0.
254
279
  */
255
280
  export async function estimateSpeechRanges(film: FootageFilm, options: Omit<EstimateFootageOptions, "isolate">): Promise<TtsResult<FootageEstimate>> {
256
281
  const estimate = await estimateFootageFilm(film, { ...options, isolate: "segments" });