@softure-ai/marketing-kit 0.14.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/CHANGELOG.md +95 -0
  2. package/README.md +136 -12
  3. package/dist/config/schema.d.ts +2 -1
  4. package/dist/config/schema.d.ts.map +1 -1
  5. package/dist/config/schema.js +7 -0
  6. package/dist/config/schema.js.map +1 -1
  7. package/dist/footage/clip-sound.d.ts +58 -0
  8. package/dist/footage/clip-sound.d.ts.map +1 -0
  9. package/dist/footage/clip-sound.js +103 -0
  10. package/dist/footage/clip-sound.js.map +1 -0
  11. package/dist/footage/declick.d.ts +1 -1
  12. package/dist/footage/declick.d.ts.map +1 -1
  13. package/dist/footage/declick.js +3 -4
  14. package/dist/footage/declick.js.map +1 -1
  15. package/dist/footage/draft.d.ts +19 -5
  16. package/dist/footage/draft.d.ts.map +1 -1
  17. package/dist/footage/draft.js +94 -24
  18. package/dist/footage/draft.js.map +1 -1
  19. package/dist/footage/elevenlabs.d.ts +5 -0
  20. package/dist/footage/elevenlabs.d.ts.map +1 -1
  21. package/dist/footage/elevenlabs.js +10 -3
  22. package/dist/footage/elevenlabs.js.map +1 -1
  23. package/dist/footage/failure.d.ts +53 -0
  24. package/dist/footage/failure.d.ts.map +1 -0
  25. package/dist/footage/failure.js +23 -0
  26. package/dist/footage/failure.js.map +1 -0
  27. package/dist/footage/fake.d.ts +10 -1
  28. package/dist/footage/fake.d.ts.map +1 -1
  29. package/dist/footage/fake.js +19 -4
  30. package/dist/footage/fake.js.map +1 -1
  31. package/dist/footage/film-mix.d.ts +5 -1
  32. package/dist/footage/film-mix.d.ts.map +1 -1
  33. package/dist/footage/film-mix.js +11 -2
  34. package/dist/footage/film-mix.js.map +1 -1
  35. package/dist/footage/hyperframes-capture.d.ts +30 -0
  36. package/dist/footage/hyperframes-capture.d.ts.map +1 -0
  37. package/dist/footage/hyperframes-capture.js +80 -0
  38. package/dist/footage/hyperframes-capture.js.map +1 -0
  39. package/dist/footage/isolate-ranges.d.ts +67 -5
  40. package/dist/footage/isolate-ranges.d.ts.map +1 -1
  41. package/dist/footage/isolate-ranges.js +92 -21
  42. package/dist/footage/isolate-ranges.js.map +1 -1
  43. package/dist/footage/loudness.d.ts +16 -2
  44. package/dist/footage/loudness.d.ts.map +1 -1
  45. package/dist/footage/loudness.js +10 -7
  46. package/dist/footage/loudness.js.map +1 -1
  47. package/dist/footage/paid.d.ts +14 -1
  48. package/dist/footage/paid.d.ts.map +1 -1
  49. package/dist/footage/paid.js +10 -4
  50. package/dist/footage/paid.js.map +1 -1
  51. package/dist/footage/pipeline.d.ts +10 -4
  52. package/dist/footage/pipeline.d.ts.map +1 -1
  53. package/dist/footage/pipeline.js +74 -64
  54. package/dist/footage/pipeline.js.map +1 -1
  55. package/dist/footage/providers.d.ts +5 -0
  56. package/dist/footage/providers.d.ts.map +1 -1
  57. package/dist/footage/providers.js.map +1 -1
  58. package/dist/footage/render.d.ts +1 -1
  59. package/dist/footage/render.d.ts.map +1 -1
  60. package/dist/footage/render.js +4 -3
  61. package/dist/footage/render.js.map +1 -1
  62. package/dist/footage/speech-ranges.d.ts +80 -14
  63. package/dist/footage/speech-ranges.d.ts.map +1 -1
  64. package/dist/footage/speech-ranges.js +146 -26
  65. package/dist/footage/speech-ranges.js.map +1 -1
  66. package/dist/index.d.ts +10 -6
  67. package/dist/index.d.ts.map +1 -1
  68. package/dist/index.js +9 -5
  69. package/dist/index.js.map +1 -1
  70. package/dist/og/index.d.ts +2 -0
  71. package/dist/og/index.d.ts.map +1 -1
  72. package/dist/og/index.js +2 -0
  73. package/dist/og/index.js.map +1 -1
  74. package/dist/og/photo-cover.d.ts +55 -0
  75. package/dist/og/photo-cover.d.ts.map +1 -0
  76. package/dist/og/photo-cover.js +215 -0
  77. package/dist/og/photo-cover.js.map +1 -0
  78. package/dist/og/photo.d.ts +51 -0
  79. package/dist/og/photo.d.ts.map +1 -0
  80. package/dist/og/photo.js +154 -0
  81. package/dist/og/photo.js.map +1 -0
  82. package/dist/voice/provider-failure.d.ts +41 -0
  83. package/dist/voice/provider-failure.d.ts.map +1 -0
  84. package/dist/voice/provider-failure.js +68 -0
  85. package/dist/voice/provider-failure.js.map +1 -0
  86. package/dist/voice/provider.d.ts +7 -1
  87. package/dist/voice/provider.d.ts.map +1 -1
  88. package/dist/voice/provider.js +1 -1
  89. package/dist/voice/provider.js.map +1 -1
  90. package/package.json +2 -1
  91. package/schema/marketing.schema.json +6 -0
  92. package/src/config/schema.ts +7 -0
  93. package/src/footage/clip-sound.ts +124 -0
  94. package/src/footage/declick.ts +3 -4
  95. package/src/footage/draft.ts +125 -24
  96. package/src/footage/elevenlabs.ts +11 -3
  97. package/src/footage/failure.ts +57 -0
  98. package/src/footage/fake.ts +28 -4
  99. package/src/footage/film-mix.ts +10 -2
  100. package/src/footage/hyperframes-capture.ts +100 -0
  101. package/src/footage/isolate-ranges.ts +158 -22
  102. package/src/footage/loudness.ts +17 -8
  103. package/src/footage/paid.ts +29 -4
  104. package/src/footage/pipeline.ts +85 -66
  105. package/src/footage/providers.ts +5 -0
  106. package/src/footage/render.ts +4 -3
  107. package/src/footage/speech-ranges.ts +197 -27
  108. package/src/index.ts +55 -1
  109. package/src/og/index.ts +23 -0
  110. package/src/og/photo-cover.ts +280 -0
  111. package/src/og/photo.ts +177 -0
  112. package/src/voice/provider-failure.ts +98 -0
  113. package/src/voice/provider.ts +7 -2
@@ -1,7 +1,11 @@
1
+ import { extname } from "node:path";
2
+
1
3
  import { footageCacheKey, getFootageCachePath } from "./cache.js";
4
+ import { getAudioCodecArgs } from "./clip-sound.js";
2
5
  import { getSoundSpan, type Edl } from "./edl.js";
3
6
  import type { ClipRole } from "./film.js";
4
7
  import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
8
+ import type { IsolationProvider } from "./providers.js";
5
9
 
6
10
  /**
7
11
  * The parts of each talking clip a cut hears, as ranges in whole milliseconds, and how their cleaned voice is laid
@@ -12,11 +16,31 @@ import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
12
16
  /** Seconds added on each side of a sound span, so a cut's cross-fade and fades read cleaned sound. */
13
17
  export const DEFAULT_SPEECH_RANGE_MARGIN = 0.25;
14
18
 
19
+ /** Seconds of the largest gap between two ranges of a clip that are isolated as one range (`planSpeechRanges`). */
20
+ export const DEFAULT_SPEECH_RANGE_MERGE_GAP = 2;
21
+
22
+ /** Seconds a range shorter than the provider's minimum is widened past it, so the cut file is safely long enough. */
23
+ export const DEFAULT_SPEECH_RANGE_MIN_PAD = 0.4;
24
+
25
+ /** Why a planned range is not sent: shorter than the provider's minimum even after merging and widening. */
26
+ export interface SpeechRangeSkip {
27
+ reason: "too_short";
28
+ /** The provider's minimum, in seconds. */
29
+ minSeconds: number;
30
+ }
31
+
15
32
  /** A range of a clip's trimmed sound, in whole milliseconds (the cache key's unit). */
16
33
  export interface SpeechRange {
17
34
  clip: string;
18
35
  startMs: number;
19
36
  endMs: number;
37
+ /**
38
+ * The part of the range the cut hears, when the range was widened with the clip's own context to reach the
39
+ * provider's minimum; absent: the whole range. Only this part of the cleaned voice is laid.
40
+ */
41
+ heard?: { startMs: number; endMs: number };
42
+ /** Set when the range cannot pass the provider (the clip itself is shorter than its minimum): it is not sent. */
43
+ skip?: SpeechRangeSkip;
20
44
  }
21
45
 
22
46
  /** Where a range's cleaned sound is laid in its clip, in milliseconds; inside the range. */
@@ -33,15 +57,35 @@ export interface SpeechRangeClip {
33
57
  duration: number;
34
58
  }
35
59
 
60
+ export interface PlanSpeechRangesOptions {
61
+ /** Seconds added on each side of a sound span; default `DEFAULT_SPEECH_RANGE_MARGIN` (0.25). */
62
+ margin?: number;
63
+ /** The isolation provider's minimum input in seconds (`IsolationProvider.minSeconds`); absent: no minimum. */
64
+ minSeconds?: number;
65
+ /**
66
+ * Ranges of a clip at most this many seconds apart are isolated as one range; default
67
+ * `DEFAULT_SPEECH_RANGE_MERGE_GAP` (2). `null` keeps every range apart (only identical ranges and ranges inside
68
+ * another are dropped); `Infinity` isolates each clip's whole used span once.
69
+ */
70
+ mergeGap?: number | null;
71
+ /** Seconds a range shorter than `minSeconds` is widened past it; default `DEFAULT_SPEECH_RANGE_MIN_PAD` (0.4). */
72
+ minPad?: number;
73
+ }
74
+
36
75
  /**
37
- * The ranges the cut hears of each talking clip: every segment's sound span (its own clip's or `audio`'s), and the
76
+ * The source ranges to isolate of each talking clip: every segment's sound span (its own clip's or `audio`'s), and the
38
77
  * picture clip's own span when `audio.ownVolume` keeps it under, widened by `margin` seconds on each side, clamped to
39
- * the clip and rounded out to whole milliseconds. Identical ranges and ranges inside another are dropped (the larger
40
- * one is laid there anyway); the rest are not merged, so a moved segment changes only its own range. Sorted by clip
41
- * order, then start.
78
+ * the clip and rounded out to whole milliseconds. Identical ranges and ranges inside another are dropped; ranges of a
79
+ * clip at most `mergeGap` seconds apart become one. With `minSeconds`, a range still shorter is widened symmetrically
80
+ * with the clip's own context to `minSeconds + minPad` (shifted where the clip's edge stops it; `heard` keeps the part
81
+ * the cut hears), ranges the widening brought within `mergeGap` are merged, and a range still shorter (a clip shorter
82
+ * than the minimum) is marked `skip`. Sorted by clip order, then start.
42
83
  */
43
- export function planSpeechRanges(edl: Edl, clips: readonly SpeechRangeClip[], options: { margin?: number } = {}): SpeechRange[] {
84
+ export function planSpeechRanges(edl: Edl, clips: readonly SpeechRangeClip[], options: PlanSpeechRangesOptions = {}): SpeechRange[] {
44
85
  const margin = Math.max(0, options.margin ?? DEFAULT_SPEECH_RANGE_MARGIN);
86
+ const mergeGap = options.mergeGap === undefined ? DEFAULT_SPEECH_RANGE_MERGE_GAP : options.mergeGap;
87
+ const gapMs = mergeGap === null ? null : Math.max(0, mergeGap) * 1000;
88
+ const minSeconds = options.minSeconds !== undefined && options.minSeconds > 0 ? options.minSeconds : null;
45
89
  const byId = new Map(clips.map((clip) => [clip.id, clip]));
46
90
  const found = new Map<string, SpeechRange[]>();
47
91
  const add = (span: { clip: string; start: number; end: number }): void => {
@@ -62,25 +106,90 @@ export function planSpeechRanges(edl: Edl, clips: readonly SpeechRangeClip[], op
62
106
  const ranges: SpeechRange[] = [];
63
107
  for (const clip of clips) {
64
108
  const list = (found.get(clip.id) ?? []).sort((a, b) => a.startMs - b.startMs || b.endMs - a.endMs);
109
+ let kept: SpeechRange[] = [];
65
110
  for (const range of list) {
66
- const isInside = ranges.some((kept) => kept.clip === range.clip && kept.startMs <= range.startMs && kept.endMs >= range.endMs);
67
- if (!isInside) ranges.push(range);
111
+ const isInside = kept.some((other) => other.startMs <= range.startMs && other.endMs >= range.endMs);
112
+ if (!isInside) kept.push(range);
68
113
  }
114
+ if (gapMs !== null) kept = mergeSpeechRanges(kept, gapMs);
115
+ if (minSeconds !== null) {
116
+ const minMs = Math.ceil(minSeconds * 1000);
117
+ const targetMs = Math.ceil((minSeconds + Math.max(0, options.minPad ?? DEFAULT_SPEECH_RANGE_MIN_PAD)) * 1000);
118
+ const clipMs = Math.floor(clip.duration * 1000);
119
+ kept = kept.map((range) => widenSpeechRange(range, { minMs, targetMs, clipMs }));
120
+ if (gapMs !== null) kept = mergeSpeechRanges(kept, gapMs);
121
+ kept = kept.map((range) => (range.endMs - range.startMs < minMs ? { ...range, skip: { reason: "too_short", minSeconds } } : range));
122
+ }
123
+ ranges.push(...kept);
69
124
  }
70
125
  return ranges;
71
126
  }
72
127
 
128
+ /** How `isolateSpeechRanges` and the estimate plan the ranges: the caller's margin and merge gap, the provider's minimum. */
129
+ export function getPlanOptions(options: Pick<PlanSpeechRangesOptions, "margin" | "mergeGap" | "minPad">, isolation: IsolationProvider): PlanSpeechRangesOptions {
130
+ return {
131
+ ...(options.margin === undefined ? {} : { margin: options.margin }),
132
+ ...(options.mergeGap === undefined ? {} : { mergeGap: options.mergeGap }),
133
+ ...(options.minPad === undefined ? {} : { minPad: options.minPad }),
134
+ ...(isolation.minSeconds === undefined ? {} : { minSeconds: isolation.minSeconds }),
135
+ };
136
+ }
137
+
138
+ /** The part of a range the cut hears. */
139
+ function getHeard(range: SpeechRange): { startMs: number; endMs: number } {
140
+ return range.heard ?? { startMs: range.startMs, endMs: range.endMs };
141
+ }
142
+
143
+ /** A range with `heard` set only when it is narrower than the range. */
144
+ function withHeard(range: Pick<SpeechRange, "clip" | "startMs" | "endMs">, heard: { startMs: number; endMs: number }): SpeechRange {
145
+ const isWhole = heard.startMs <= range.startMs && heard.endMs >= range.endMs;
146
+ return isWhole ? { clip: range.clip, startMs: range.startMs, endMs: range.endMs } : { clip: range.clip, startMs: range.startMs, endMs: range.endMs, heard };
147
+ }
148
+
149
+ /** One clip's ranges (sorted by start) with every two at most `gapMs` apart merged into one; the heard parts joined. */
150
+ function mergeSpeechRanges(ranges: readonly SpeechRange[], gapMs: number): SpeechRange[] {
151
+ const merged: SpeechRange[] = [];
152
+ for (const range of [...ranges].sort((a, b) => a.startMs - b.startMs)) {
153
+ const last = merged.at(-1);
154
+ if (last === undefined || range.startMs - last.endMs > gapMs) {
155
+ merged.push(range);
156
+ continue;
157
+ }
158
+ const lastHeard = getHeard(last);
159
+ const heard = getHeard(range);
160
+ merged[merged.length - 1] = withHeard(
161
+ { clip: last.clip, startMs: last.startMs, endMs: Math.max(last.endMs, range.endMs) },
162
+ { startMs: Math.min(lastHeard.startMs, heard.startMs), endMs: Math.max(lastHeard.endMs, heard.endMs) },
163
+ );
164
+ }
165
+ return merged;
166
+ }
167
+
168
+ /** A range shorter than `minMs` widened symmetrically to `targetMs` with the clip's own context, inside the clip. */
169
+ function widenSpeechRange(range: SpeechRange, limits: { minMs: number; targetMs: number; clipMs: number }): SpeechRange {
170
+ const length = range.endMs - range.startMs;
171
+ if (length >= limits.minMs) return range;
172
+ const target = Math.min(limits.targetMs, limits.clipMs);
173
+ const endMs = Math.min(limits.clipMs, Math.max(0, range.startMs - Math.floor((target - length) / 2)) + target);
174
+ const startMs = Math.max(0, endMs - target);
175
+ return withHeard({ clip: range.clip, startMs, endMs }, getHeard(range));
176
+ }
177
+
73
178
  /**
74
- * Where each of one clip's ranges is laid: the whole range, except that two overlapping neighbours switch at the
75
- * middle of their overlap. Takes ranges of one clip, none inside another (as `planSpeechRanges` answers them).
179
+ * Where each of one clip's ranges is laid: the part the cut hears (`heard`, else the whole range), except that two
180
+ * overlapping neighbours switch at the middle of their overlap. Takes ranges of one clip, none inside another (as
181
+ * `planSpeechRanges` answers them).
76
182
  */
77
183
  export function getSpeechRangeWindows(ranges: readonly SpeechRange[]): SpeechRangeWindow[] {
78
- const sorted = [...ranges].sort((a, b) => a.startMs - b.startMs);
184
+ const sorted = [...ranges].sort((a, b) => getHeard(a).startMs - getHeard(b).startMs);
79
185
  return sorted.map((range, index) => {
186
+ const heard = getHeard(range);
80
187
  const before = sorted[index - 1];
81
188
  const after = sorted[index + 1];
82
- const startMs = before !== undefined && before.endMs > range.startMs ? Math.round((range.startMs + before.endMs) / 2) : range.startMs;
83
- const endMs = after !== undefined && after.startMs < range.endMs ? Math.round((after.startMs + range.endMs) / 2) : range.endMs;
189
+ const beforeEnd = before === undefined ? null : getHeard(before).endMs;
190
+ const afterStart = after === undefined ? null : getHeard(after).startMs;
191
+ const startMs = beforeEnd !== null && beforeEnd > heard.startMs ? Math.round((heard.startMs + beforeEnd) / 2) : heard.startMs;
192
+ const endMs = afterStart !== null && afterStart < heard.endMs ? Math.round((afterStart + heard.endMs) / 2) : heard.endMs;
84
193
  return { range, startMs, endMs };
85
194
  });
86
195
  }
@@ -91,11 +200,18 @@ export function getSpeechRangeCache(filmId: string, clipKey: string, range: Spee
91
200
  return { key, path: getFootageCachePath(options.cacheDir, filmId, "isolated", key) };
92
201
  }
93
202
 
203
+ /** A range's paid cache entry, the app's own: its key (reported on the charge) and file. `clipKey` is the clip's identity. */
204
+ export type SpeechRangeCacheResolver = (range: SpeechRange, clipKey: string) => { key: string; path: string };
205
+
94
206
  const seconds = (ms: number): string => (ms / 1000).toFixed(3);
95
207
 
96
- /** One range of a clip's sound (`audio`, read from its start) as mp3, for the isolation provider. */
208
+ /**
209
+ * One range of a clip's sound (`audio`, read from its start) for the isolation provider: 24-bit PCM for a `.wav`
210
+ * output (lossless, issue #1139), else as `getAudioCodecArgs` encodes it.
211
+ */
97
212
  export function buildRangeAudioArgs(audio: string, range: Pick<SpeechRange, "startMs" | "endMs">, output: string): string[] {
98
- return ["-v", "error", "-y", "-ss", seconds(range.startMs), "-t", seconds(range.endMs - range.startMs), "-i", audio, "-vn", "-c:a", "libmp3lame", "-q:a", "2", output];
213
+ const codec = extname(output).toLowerCase() === ".wav" ? ["-c:a", "pcm_s24le"] : getAudioCodecArgs(output);
214
+ return ["-v", "error", "-y", "-ss", seconds(range.startMs), "-t", seconds(range.endMs - range.startMs), "-i", audio, "-vn", ...codec, output];
99
215
  }
100
216
 
101
217
  /** A cleaned range laid back: its file, and whether it is mono (panned to both channels at full level). */
@@ -105,27 +221,81 @@ export interface LaidRange {
105
221
  isMono: boolean;
106
222
  }
107
223
 
224
+ /** Milliseconds of the linear crossfade at every join between the clip's own sound and a cleaned window. */
225
+ export const DEFAULT_SPEECH_RANGE_FADE_MS = 15;
226
+
227
+ /** Where a window's cleaned sound fades in and out, in milliseconds of the clip: `inStart ≤ inEnd ≤ outStart ≤ outEnd`. */
228
+ export interface SpeechRangeFade {
229
+ inStartMs: number;
230
+ inEndMs: number;
231
+ outStartMs: number;
232
+ outEndMs: number;
233
+ }
234
+
235
+ /**
236
+ * The crossfades of one clip's windows (as `getSpeechRangeWindows` answers them). At a switch between two overlapping
237
+ * neighbours both ranges have sound, so the fade is centred on the switch, widened by half the fade on each side (at
238
+ * most half the overlap) and shared by both windows, whose gains then sum to one. At a range's own edge the range has
239
+ * no sound outside it, so the fade lies inside the window. No fade is longer than half its window.
240
+ */
241
+ export function getSpeechRangeFades(windows: readonly SpeechRangeWindow[], fadeMs: number): SpeechRangeFade[] {
242
+ const fade = Math.max(0, fadeMs);
243
+ const sorted = [...windows].sort((a, b) => a.startMs - b.startMs);
244
+ const halfWindow = (window: SpeechRangeWindow): number => (window.endMs - window.startMs) / 2;
245
+ /** Half the shared fade at the switch between `before` and `after`; null when they do not overlap. */
246
+ const getSwitchHalf = (before: SpeechRangeWindow | undefined, after: SpeechRangeWindow | undefined): number | null => {
247
+ if (before === undefined || after === undefined || before.range.endMs <= after.range.startMs || before.endMs !== after.startMs) return null;
248
+ const room = Math.min(before.range.endMs - before.endMs, after.startMs - after.range.startMs);
249
+ return Math.floor(Math.min(fade / 2, room, halfWindow(before) / 2, halfWindow(after) / 2));
250
+ };
251
+ return sorted.map((window, index) => {
252
+ const edge = Math.floor(Math.min(fade, halfWindow(window)));
253
+ const startHalf = getSwitchHalf(sorted[index - 1], window);
254
+ const endHalf = getSwitchHalf(window, sorted[index + 1]);
255
+ const [inStartMs, inEndMs] = startHalf === null ? [window.startMs, window.startMs + edge] : [window.startMs - startHalf, window.startMs + startHalf];
256
+ const [outStartMs, outEndMs] = endHalf === null ? [window.endMs - edge, window.endMs] : [window.endMs - endHalf, window.endMs + endHalf];
257
+ return { inStartMs, inEndMs, outStartMs, outEndMs };
258
+ });
259
+ }
260
+
261
+ /** The gain of a window with `fade` at time `t`, as an ffmpeg expression: 0 outside, 1 inside, linear in the fades. */
262
+ function buildGainExpression(fade: SpeechRangeFade): string {
263
+ const rise = fade.inEndMs > fade.inStartMs ? `clip((t-${seconds(fade.inStartMs)})/${seconds(fade.inEndMs - fade.inStartMs)},0,1)` : `gte(t,${seconds(fade.inStartMs)})`;
264
+ const fall = fade.outEndMs > fade.outStartMs ? `clip((${seconds(fade.outEndMs)}-t)/${seconds(fade.outEndMs - fade.outStartMs)},0,1)` : `lt(t,${seconds(fade.outEndMs)})`;
265
+ return `min(${rise},${fall})`;
266
+ }
267
+
108
268
  /**
109
- * The clip's own sound with each window replaced by its cleaned range: the base muted inside every window, each
110
- * window cut from its cleaned file and delayed into place, summed without level change. As long as the base; 48 kHz
111
- * stereo like the speech track; a mono answer goes to both channels as it is, not up-mixed at -3 dB.
269
+ * The clip's own sound with each window replaced by its cleaned range, crossfaded at every join (`fadeMs`, default
270
+ * `DEFAULT_SPEECH_RANGE_FADE_MS`; 0 switches at once): each window is cut from its cleaned file to its fade bounds,
271
+ * faded in and out linearly and delayed into place in samples; the base is multiplied by one minus the windows'
272
+ * gains, so the sum keeps its level. As long as the base; 48 kHz stereo like the speech track; a mono answer goes to
273
+ * both channels as it is, not up-mixed at -3 dB.
112
274
  */
113
- export function buildRangeLayArgs(base: string, laid: readonly LaidRange[], output: string): string[] {
275
+ export function buildRangeLayArgs(base: string, laid: readonly LaidRange[], output: string, options: { fadeMs?: number } = {}): string[] {
114
276
  const format = `aformat=sample_rates=${FOOTAGE_SAMPLE_RATE}:channel_layouts=stereo`;
115
- if (laid.length === 0) return ["-v", "error", "-y", "-i", base, "-vn", "-af", format, "-c:a", "libmp3lame", "-q:a", "2", output];
277
+ if (laid.length === 0) return ["-v", "error", "-y", "-i", base, "-vn", "-af", format, ...getAudioCodecArgs(output), output];
278
+ const windows = laid.map(({ window }) => window);
279
+ const fades = getSpeechRangeFades(windows, options.fadeMs ?? DEFAULT_SPEECH_RANGE_FADE_MS);
280
+ const byWindow = new Map([...windows].sort((a, b) => a.startMs - b.startMs).map((window, index) => [window, fades[index]]));
116
281
  const inputs = ["-i", base];
117
- const mute = laid.map(({ window }) => `between(t,${seconds(window.startMs)},${seconds(window.endMs)})`).join("+");
118
- const parts = [`[0:a]${format},volume=0:enable='${mute}'[base]`];
282
+ const gains: string[] = [];
283
+ const parts: string[] = [];
119
284
  laid.forEach(({ window, file, isMono }, index) => {
285
+ const fade = byWindow.get(window);
286
+ if (fade === undefined) return;
120
287
  inputs.push("-i", file);
121
- const offset = window.startMs - window.range.startMs;
122
- const length = window.endMs - window.startMs;
288
+ gains.push(buildGainExpression(fade));
123
289
  const channels = isMono ? "pan=stereo|c0=c0|c1=c0," : "";
290
+ const fadeIn = fade.inEndMs > fade.inStartMs ? `,afade=t=in:st=0:d=${seconds(fade.inEndMs - fade.inStartMs)}` : "";
291
+ const fadeOut = fade.outEndMs > fade.outStartMs ? `,afade=t=out:st=${seconds(fade.outStartMs - fade.inStartMs)}:d=${seconds(fade.outEndMs - fade.outStartMs)}` : "";
292
+ const delay = Math.round((fade.inStartMs * FOOTAGE_SAMPLE_RATE) / 1000);
124
293
  parts.push(
125
- `[${index + 1}:a]atrim=start=${seconds(offset)}:end=${seconds(offset + length)},asetpts=PTS-STARTPTS,${channels}${format},adelay=${window.startMs}:all=1[w${index}]`,
294
+ `[${index + 1}:a]atrim=start=${seconds(fade.inStartMs - window.range.startMs)}:end=${seconds(fade.outEndMs - window.range.startMs)},asetpts=PTS-STARTPTS,${channels}${format}${fadeIn}${fadeOut},adelay=${String(delay)}S:all=1[w${String(index)}]`,
126
295
  );
127
296
  });
128
- const mixed = laid.map((_, index) => `[w${index}]`).join("");
129
- parts.push(`[base]${mixed}amix=inputs=${laid.length + 1}:duration=first:normalize=0[speech]`);
130
- return ["-v", "error", "-y", ...inputs, "-filter_complex", parts.join(";"), "-map", "[speech]", "-c:a", "libmp3lame", "-q:a", "2", output];
297
+ parts.unshift(`[0:a]${format},aeval='val(ch)*(1-(${gains.join("+")}))':c=same[base]`);
298
+ const mixed = laid.map((_, index) => `[w${String(index)}]`).join("");
299
+ parts.push(`[base]${mixed}amix=inputs=${String(laid.length + 1)}:duration=first:normalize=0[speech]`);
300
+ return ["-v", "error", "-y", ...inputs, "-filter_complex", parts.join(";"), "-map", "[speech]", ...getAudioCodecArgs(output), output];
131
301
  }
package/src/index.ts CHANGED
@@ -84,6 +84,17 @@ export {
84
84
  type TtsRequest,
85
85
  } from "./voice/voiceover.js";
86
86
  export type { TtsEstimate, TtsInput, TtsProvider, TtsRecording, TtsResult } from "./voice/provider.js";
87
+ export {
88
+ describeProviderFailure,
89
+ getConnectionFailure,
90
+ getLocalRefusal,
91
+ getProviderFailure,
92
+ getProviderFailureKind,
93
+ readProviderErrorBody,
94
+ PROVIDER_MESSAGE_LENGTH,
95
+ type ProviderFailure,
96
+ type ProviderFailureKind,
97
+ } from "./voice/provider-failure.js";
87
98
  export { ELEVENLABS_API_KEY_ENV, ELEVENLABS_PROVIDER_ID, createElevenLabsProvider, type ElevenLabsOptions } from "./voice/elevenlabs.js";
88
99
  export { createFakeTtsProvider, type FakeTtsOptions, type FakeTtsProvider } from "./voice/fake.js";
89
100
  export { createTtsProvider, type CreateTtsProviderOptions, type TtsProviderId } from "./voice/providers.js";
@@ -156,11 +167,14 @@ export {
156
167
  loadOgFonts,
157
168
  renderConfiguredOgImage,
158
169
  renderOgImage,
170
+ renderPhotoCover,
159
171
  type ConfiguredOgImageOptions,
160
172
  type OgBrand,
161
173
  type OgImageInput,
162
174
  type OgResult,
163
175
  type OgTemplateId,
176
+ type PhotoCover,
177
+ type PhotoCoverInput,
164
178
  } from "./og/index.js";
165
179
  export { buildPosts, postsMarkdown, type PlatformPost, type PostsInput } from "./posts/posts.js";
166
180
  export { createActionScene, getLocator, runAction, type LocatorSource } from "./record/actions.js";
@@ -293,6 +307,7 @@ export {
293
307
  type SttTranscript,
294
308
  } from "./footage/providers.js";
295
309
  export {
310
+ ELEVENLABS_ISOLATION_MIN_SECONDS,
296
311
  ELEVENLABS_MUSIC_MODEL,
297
312
  ELEVENLABS_STT_MODEL,
298
313
  createElevenLabsFootageProviders,
@@ -309,6 +324,7 @@ export {
309
324
  createFakeMusicProvider,
310
325
  createFakeSttProvider,
311
326
  makePlaceholderTranscript,
327
+ type FakeIsolationOptions,
312
328
  type FakeIsolationProvider,
313
329
  type FakeMusicProvider,
314
330
  type FakeSttOptions,
@@ -419,6 +435,18 @@ export {
419
435
  type PicturePieceArgsInput,
420
436
  } from "./footage/picture.js";
421
437
  export {
438
+ SPEECH_PRE_GAIN,
439
+ buildClipSoundArgs,
440
+ buildClipSoundMeasureArgs,
441
+ getAudioCodecArgs,
442
+ getClipSoundPath,
443
+ getPreGainDb,
444
+ prepareClipSounds,
445
+ type ClipSound,
446
+ type ClipSoundInput,
447
+ } from "./footage/clip-sound.js";
448
+ export {
449
+ AUDIO_BITRATE_RANGE_KBPS,
422
450
  DEFAULT_LOUDNESS,
423
451
  buildLoudnessArgs,
424
452
  buildLoudnessMeasureArgs,
@@ -501,18 +529,43 @@ export {
501
529
  type FootageEstimateItem,
502
530
  } from "./footage/paid.js";
503
531
  export {
532
+ DEFAULT_SPEECH_RANGE_FADE_MS,
504
533
  DEFAULT_SPEECH_RANGE_MARGIN,
534
+ DEFAULT_SPEECH_RANGE_MERGE_GAP,
535
+ DEFAULT_SPEECH_RANGE_MIN_PAD,
505
536
  buildRangeAudioArgs,
506
537
  buildRangeLayArgs,
507
538
  getSpeechRangeCache,
539
+ getSpeechRangeFades,
508
540
  getSpeechRangeWindows,
509
541
  planSpeechRanges,
510
542
  type LaidRange,
543
+ type PlanSpeechRangesOptions,
511
544
  type SpeechRange,
545
+ type SpeechRangeCacheResolver,
512
546
  type SpeechRangeClip,
547
+ type SpeechRangeFade,
548
+ type SpeechRangeSkip,
513
549
  type SpeechRangeWindow,
514
550
  } from "./footage/speech-ranges.js";
515
- export { isolateSpeechRanges, type IsolateSpeechRangesOptions, type IsolatedSpeech, type IsolatedSpeechRange } from "./footage/isolate-ranges.js";
551
+ export {
552
+ DEFAULT_SPEECH_RANGE_ATTEMPTS,
553
+ DEFAULT_SPEECH_RANGE_RETRY_DELAYS_MS,
554
+ isolateSpeechRanges,
555
+ type IsolateSpeechRangesOptions,
556
+ type IsolatedSpeech,
557
+ type IsolatedSpeechRange,
558
+ type SpeechRangeOutcome,
559
+ type SpeechRangeRetry,
560
+ } from "./footage/isolate-ranges.js";
561
+ export {
562
+ HYPERFRAMES_CAPTURE_STAGES,
563
+ readHyperframesCapture,
564
+ type HyperframesCapture,
565
+ type HyperframesCapturePath,
566
+ type HyperframesCaptureStage,
567
+ type HyperframesGpuMode,
568
+ } from "./footage/hyperframes-capture.js";
516
569
  export { RENDER_MIN_FREE_BYTES, findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./footage/disk.js";
517
570
  export { DRAFT_FILTERS, REQUIRED_DECODERS, REQUIRED_FILTERS, checkFootageTools, findBrowserProblem, readFfmpegListing, type FootageToolsOptions } from "./footage/tools.js";
518
571
  export {
@@ -546,6 +599,7 @@ export {
546
599
  type RenderedFootage,
547
600
  type TranscribeOutcome,
548
601
  } from "./footage/pipeline.js";
602
+ export { getProviderFootageFailure, isRetryableFootageFailure, type FootageError, type FootageFailure, type FootageFailureKind } from "./footage/failure.js";
549
603
  export {
550
604
  formatMessage,
551
605
  getMarketingMessages,
package/src/og/index.ts CHANGED
@@ -18,6 +18,29 @@ export {
18
18
  type OgBrand,
19
19
  type OgImageInput,
20
20
  } from "./render.js";
21
+ export {
22
+ MAX_COVER_CAPTION_LENGTH,
23
+ PHOTO_COVER_QUALITY,
24
+ PHOTO_COVER_SIZE,
25
+ getCoverBand,
26
+ getCoverCaptionSize,
27
+ renderPhotoCover,
28
+ type CoverBand,
29
+ type PhotoCover,
30
+ type PhotoCoverInput,
31
+ } from "./photo-cover.js";
32
+ export {
33
+ getCoverCrop,
34
+ getOrientationMatrix,
35
+ getOrientedSize,
36
+ readPhotoInfo,
37
+ readTiffOrientation,
38
+ type CoverCrop,
39
+ type FocusPoint,
40
+ type PhotoFormat,
41
+ type PhotoInfo,
42
+ type PhotoOrientation,
43
+ } from "./photo.js";
21
44
  export { OG_TEMPLATE_IDS, OG_TEMPLATES, isOgTemplateId, type OgTemplateId } from "./templates/index.js";
22
45
  export { PORTRAIT_BASE, getLayoutScale, type OgLayout, type OgTemplateContext } from "./templates/context.js";
23
46
  export {