@softure-ai/marketing-kit 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +95 -0
- package/README.md +136 -12
- package/dist/config/schema.d.ts +2 -1
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +7 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/footage/clip-sound.d.ts +58 -0
- package/dist/footage/clip-sound.d.ts.map +1 -0
- package/dist/footage/clip-sound.js +103 -0
- package/dist/footage/clip-sound.js.map +1 -0
- package/dist/footage/declick.d.ts +1 -1
- package/dist/footage/declick.d.ts.map +1 -1
- package/dist/footage/declick.js +3 -4
- package/dist/footage/declick.js.map +1 -1
- package/dist/footage/draft.d.ts +19 -5
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +94 -24
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/elevenlabs.d.ts +5 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -1
- package/dist/footage/elevenlabs.js +10 -3
- package/dist/footage/elevenlabs.js.map +1 -1
- package/dist/footage/failure.d.ts +53 -0
- package/dist/footage/failure.d.ts.map +1 -0
- package/dist/footage/failure.js +23 -0
- package/dist/footage/failure.js.map +1 -0
- package/dist/footage/fake.d.ts +10 -1
- package/dist/footage/fake.d.ts.map +1 -1
- package/dist/footage/fake.js +19 -4
- package/dist/footage/fake.js.map +1 -1
- package/dist/footage/film-mix.d.ts +5 -1
- package/dist/footage/film-mix.d.ts.map +1 -1
- package/dist/footage/film-mix.js +11 -2
- package/dist/footage/film-mix.js.map +1 -1
- package/dist/footage/hyperframes-capture.d.ts +30 -0
- package/dist/footage/hyperframes-capture.d.ts.map +1 -0
- package/dist/footage/hyperframes-capture.js +80 -0
- package/dist/footage/hyperframes-capture.js.map +1 -0
- package/dist/footage/isolate-ranges.d.ts +67 -5
- package/dist/footage/isolate-ranges.d.ts.map +1 -1
- package/dist/footage/isolate-ranges.js +92 -21
- package/dist/footage/isolate-ranges.js.map +1 -1
- package/dist/footage/loudness.d.ts +16 -2
- package/dist/footage/loudness.d.ts.map +1 -1
- package/dist/footage/loudness.js +10 -7
- package/dist/footage/loudness.js.map +1 -1
- package/dist/footage/paid.d.ts +14 -1
- package/dist/footage/paid.d.ts.map +1 -1
- package/dist/footage/paid.js +10 -4
- package/dist/footage/paid.js.map +1 -1
- package/dist/footage/pipeline.d.ts +10 -4
- package/dist/footage/pipeline.d.ts.map +1 -1
- package/dist/footage/pipeline.js +74 -64
- package/dist/footage/pipeline.js.map +1 -1
- package/dist/footage/providers.d.ts +5 -0
- package/dist/footage/providers.d.ts.map +1 -1
- package/dist/footage/providers.js.map +1 -1
- package/dist/footage/render.d.ts +1 -1
- package/dist/footage/render.d.ts.map +1 -1
- package/dist/footage/render.js +4 -3
- package/dist/footage/render.js.map +1 -1
- package/dist/footage/speech-ranges.d.ts +80 -14
- package/dist/footage/speech-ranges.d.ts.map +1 -1
- package/dist/footage/speech-ranges.js +146 -26
- package/dist/footage/speech-ranges.js.map +1 -1
- package/dist/index.d.ts +10 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -5
- package/dist/index.js.map +1 -1
- package/dist/og/index.d.ts +2 -0
- package/dist/og/index.d.ts.map +1 -1
- package/dist/og/index.js +2 -0
- package/dist/og/index.js.map +1 -1
- package/dist/og/photo-cover.d.ts +55 -0
- package/dist/og/photo-cover.d.ts.map +1 -0
- package/dist/og/photo-cover.js +215 -0
- package/dist/og/photo-cover.js.map +1 -0
- package/dist/og/photo.d.ts +51 -0
- package/dist/og/photo.d.ts.map +1 -0
- package/dist/og/photo.js +154 -0
- package/dist/og/photo.js.map +1 -0
- package/dist/voice/provider-failure.d.ts +41 -0
- package/dist/voice/provider-failure.d.ts.map +1 -0
- package/dist/voice/provider-failure.js +68 -0
- package/dist/voice/provider-failure.js.map +1 -0
- package/dist/voice/provider.d.ts +7 -1
- package/dist/voice/provider.d.ts.map +1 -1
- package/dist/voice/provider.js +1 -1
- package/dist/voice/provider.js.map +1 -1
- package/package.json +2 -1
- package/schema/marketing.schema.json +6 -0
- package/src/config/schema.ts +7 -0
- package/src/footage/clip-sound.ts +124 -0
- package/src/footage/declick.ts +3 -4
- package/src/footage/draft.ts +125 -24
- package/src/footage/elevenlabs.ts +11 -3
- package/src/footage/failure.ts +57 -0
- package/src/footage/fake.ts +28 -4
- package/src/footage/film-mix.ts +10 -2
- package/src/footage/hyperframes-capture.ts +100 -0
- package/src/footage/isolate-ranges.ts +158 -22
- package/src/footage/loudness.ts +17 -8
- package/src/footage/paid.ts +29 -4
- package/src/footage/pipeline.ts +85 -66
- package/src/footage/providers.ts +5 -0
- package/src/footage/render.ts +4 -3
- package/src/footage/speech-ranges.ts +197 -27
- package/src/index.ts +55 -1
- package/src/og/index.ts +23 -0
- package/src/og/photo-cover.ts +280 -0
- package/src/og/photo.ts +177 -0
- package/src/voice/provider-failure.ts +98 -0
- package/src/voice/provider.ts +7 -2
|
@@ -1,7 +1,11 @@
|
|
|
1
|
+
import { extname } from "node:path";
|
|
2
|
+
|
|
1
3
|
import { footageCacheKey, getFootageCachePath } from "./cache.js";
|
|
4
|
+
import { getAudioCodecArgs } from "./clip-sound.js";
|
|
2
5
|
import { getSoundSpan, type Edl } from "./edl.js";
|
|
3
6
|
import type { ClipRole } from "./film.js";
|
|
4
7
|
import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
8
|
+
import type { IsolationProvider } from "./providers.js";
|
|
5
9
|
|
|
6
10
|
/**
|
|
7
11
|
* The parts of each talking clip a cut hears, as ranges in whole milliseconds, and how their cleaned voice is laid
|
|
@@ -12,11 +16,31 @@ import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
|
|
|
12
16
|
/** Seconds added on each side of a sound span, so a cut's cross-fade and fades read cleaned sound. */
|
|
13
17
|
export const DEFAULT_SPEECH_RANGE_MARGIN = 0.25;
|
|
14
18
|
|
|
19
|
+
/** Seconds of the largest gap between two ranges of a clip that are isolated as one range (`planSpeechRanges`). */
|
|
20
|
+
export const DEFAULT_SPEECH_RANGE_MERGE_GAP = 2;
|
|
21
|
+
|
|
22
|
+
/** Seconds a range shorter than the provider's minimum is widened past it, so the cut file is safely long enough. */
|
|
23
|
+
export const DEFAULT_SPEECH_RANGE_MIN_PAD = 0.4;
|
|
24
|
+
|
|
25
|
+
/** Why a planned range is not sent: shorter than the provider's minimum even after merging and widening. */
|
|
26
|
+
export interface SpeechRangeSkip {
|
|
27
|
+
reason: "too_short";
|
|
28
|
+
/** The provider's minimum, in seconds. */
|
|
29
|
+
minSeconds: number;
|
|
30
|
+
}
|
|
31
|
+
|
|
15
32
|
/** A range of a clip's trimmed sound, in whole milliseconds (the cache key's unit). */
|
|
16
33
|
export interface SpeechRange {
|
|
17
34
|
clip: string;
|
|
18
35
|
startMs: number;
|
|
19
36
|
endMs: number;
|
|
37
|
+
/**
|
|
38
|
+
* The part of the range the cut hears, when the range was widened with the clip's own context to reach the
|
|
39
|
+
* provider's minimum; absent: the whole range. Only this part of the cleaned voice is laid.
|
|
40
|
+
*/
|
|
41
|
+
heard?: { startMs: number; endMs: number };
|
|
42
|
+
/** Set when the range cannot pass the provider (the clip itself is shorter than its minimum): it is not sent. */
|
|
43
|
+
skip?: SpeechRangeSkip;
|
|
20
44
|
}
|
|
21
45
|
|
|
22
46
|
/** Where a range's cleaned sound is laid in its clip, in milliseconds; inside the range. */
|
|
@@ -33,15 +57,35 @@ export interface SpeechRangeClip {
|
|
|
33
57
|
duration: number;
|
|
34
58
|
}
|
|
35
59
|
|
|
60
|
+
export interface PlanSpeechRangesOptions {
|
|
61
|
+
/** Seconds added on each side of a sound span; default `DEFAULT_SPEECH_RANGE_MARGIN` (0.25). */
|
|
62
|
+
margin?: number;
|
|
63
|
+
/** The isolation provider's minimum input in seconds (`IsolationProvider.minSeconds`); absent: no minimum. */
|
|
64
|
+
minSeconds?: number;
|
|
65
|
+
/**
|
|
66
|
+
* Ranges of a clip at most this many seconds apart are isolated as one range; default
|
|
67
|
+
* `DEFAULT_SPEECH_RANGE_MERGE_GAP` (2). `null` keeps every range apart (only identical ranges and ranges inside
|
|
68
|
+
* another are dropped); `Infinity` isolates each clip's whole used span once.
|
|
69
|
+
*/
|
|
70
|
+
mergeGap?: number | null;
|
|
71
|
+
/** Seconds a range shorter than `minSeconds` is widened past it; default `DEFAULT_SPEECH_RANGE_MIN_PAD` (0.4). */
|
|
72
|
+
minPad?: number;
|
|
73
|
+
}
|
|
74
|
+
|
|
36
75
|
/**
|
|
37
|
-
* The ranges
|
|
76
|
+
* The source ranges to isolate of each talking clip: every segment's sound span (its own clip's or `audio`'s), and the
|
|
38
77
|
* picture clip's own span when `audio.ownVolume` keeps it under, widened by `margin` seconds on each side, clamped to
|
|
39
|
-
* the clip and rounded out to whole milliseconds. Identical ranges and ranges inside another are dropped
|
|
40
|
-
*
|
|
41
|
-
*
|
|
78
|
+
* the clip and rounded out to whole milliseconds. Identical ranges and ranges inside another are dropped; ranges of a
|
|
79
|
+
* clip at most `mergeGap` seconds apart become one. With `minSeconds`, a range still shorter is widened symmetrically
|
|
80
|
+
* with the clip's own context to `minSeconds + minPad` (shifted where the clip's edge stops it; `heard` keeps the part
|
|
81
|
+
* the cut hears), ranges the widening brought within `mergeGap` are merged, and a range still shorter (a clip shorter
|
|
82
|
+
* than the minimum) is marked `skip`. Sorted by clip order, then start.
|
|
42
83
|
*/
|
|
43
|
-
export function planSpeechRanges(edl: Edl, clips: readonly SpeechRangeClip[], options:
|
|
84
|
+
export function planSpeechRanges(edl: Edl, clips: readonly SpeechRangeClip[], options: PlanSpeechRangesOptions = {}): SpeechRange[] {
|
|
44
85
|
const margin = Math.max(0, options.margin ?? DEFAULT_SPEECH_RANGE_MARGIN);
|
|
86
|
+
const mergeGap = options.mergeGap === undefined ? DEFAULT_SPEECH_RANGE_MERGE_GAP : options.mergeGap;
|
|
87
|
+
const gapMs = mergeGap === null ? null : Math.max(0, mergeGap) * 1000;
|
|
88
|
+
const minSeconds = options.minSeconds !== undefined && options.minSeconds > 0 ? options.minSeconds : null;
|
|
45
89
|
const byId = new Map(clips.map((clip) => [clip.id, clip]));
|
|
46
90
|
const found = new Map<string, SpeechRange[]>();
|
|
47
91
|
const add = (span: { clip: string; start: number; end: number }): void => {
|
|
@@ -62,25 +106,90 @@ export function planSpeechRanges(edl: Edl, clips: readonly SpeechRangeClip[], op
|
|
|
62
106
|
const ranges: SpeechRange[] = [];
|
|
63
107
|
for (const clip of clips) {
|
|
64
108
|
const list = (found.get(clip.id) ?? []).sort((a, b) => a.startMs - b.startMs || b.endMs - a.endMs);
|
|
109
|
+
let kept: SpeechRange[] = [];
|
|
65
110
|
for (const range of list) {
|
|
66
|
-
const isInside =
|
|
67
|
-
if (!isInside)
|
|
111
|
+
const isInside = kept.some((other) => other.startMs <= range.startMs && other.endMs >= range.endMs);
|
|
112
|
+
if (!isInside) kept.push(range);
|
|
68
113
|
}
|
|
114
|
+
if (gapMs !== null) kept = mergeSpeechRanges(kept, gapMs);
|
|
115
|
+
if (minSeconds !== null) {
|
|
116
|
+
const minMs = Math.ceil(minSeconds * 1000);
|
|
117
|
+
const targetMs = Math.ceil((minSeconds + Math.max(0, options.minPad ?? DEFAULT_SPEECH_RANGE_MIN_PAD)) * 1000);
|
|
118
|
+
const clipMs = Math.floor(clip.duration * 1000);
|
|
119
|
+
kept = kept.map((range) => widenSpeechRange(range, { minMs, targetMs, clipMs }));
|
|
120
|
+
if (gapMs !== null) kept = mergeSpeechRanges(kept, gapMs);
|
|
121
|
+
kept = kept.map((range) => (range.endMs - range.startMs < minMs ? { ...range, skip: { reason: "too_short", minSeconds } } : range));
|
|
122
|
+
}
|
|
123
|
+
ranges.push(...kept);
|
|
69
124
|
}
|
|
70
125
|
return ranges;
|
|
71
126
|
}
|
|
72
127
|
|
|
128
|
+
/** How `isolateSpeechRanges` and the estimate plan the ranges: the caller's margin and merge gap, the provider's minimum. */
|
|
129
|
+
export function getPlanOptions(options: Pick<PlanSpeechRangesOptions, "margin" | "mergeGap" | "minPad">, isolation: IsolationProvider): PlanSpeechRangesOptions {
|
|
130
|
+
return {
|
|
131
|
+
...(options.margin === undefined ? {} : { margin: options.margin }),
|
|
132
|
+
...(options.mergeGap === undefined ? {} : { mergeGap: options.mergeGap }),
|
|
133
|
+
...(options.minPad === undefined ? {} : { minPad: options.minPad }),
|
|
134
|
+
...(isolation.minSeconds === undefined ? {} : { minSeconds: isolation.minSeconds }),
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** The part of a range the cut hears. */
|
|
139
|
+
function getHeard(range: SpeechRange): { startMs: number; endMs: number } {
|
|
140
|
+
return range.heard ?? { startMs: range.startMs, endMs: range.endMs };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** A range with `heard` set only when it is narrower than the range. */
|
|
144
|
+
function withHeard(range: Pick<SpeechRange, "clip" | "startMs" | "endMs">, heard: { startMs: number; endMs: number }): SpeechRange {
|
|
145
|
+
const isWhole = heard.startMs <= range.startMs && heard.endMs >= range.endMs;
|
|
146
|
+
return isWhole ? { clip: range.clip, startMs: range.startMs, endMs: range.endMs } : { clip: range.clip, startMs: range.startMs, endMs: range.endMs, heard };
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** One clip's ranges (sorted by start) with every two at most `gapMs` apart merged into one; the heard parts joined. */
|
|
150
|
+
function mergeSpeechRanges(ranges: readonly SpeechRange[], gapMs: number): SpeechRange[] {
|
|
151
|
+
const merged: SpeechRange[] = [];
|
|
152
|
+
for (const range of [...ranges].sort((a, b) => a.startMs - b.startMs)) {
|
|
153
|
+
const last = merged.at(-1);
|
|
154
|
+
if (last === undefined || range.startMs - last.endMs > gapMs) {
|
|
155
|
+
merged.push(range);
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
const lastHeard = getHeard(last);
|
|
159
|
+
const heard = getHeard(range);
|
|
160
|
+
merged[merged.length - 1] = withHeard(
|
|
161
|
+
{ clip: last.clip, startMs: last.startMs, endMs: Math.max(last.endMs, range.endMs) },
|
|
162
|
+
{ startMs: Math.min(lastHeard.startMs, heard.startMs), endMs: Math.max(lastHeard.endMs, heard.endMs) },
|
|
163
|
+
);
|
|
164
|
+
}
|
|
165
|
+
return merged;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** A range shorter than `minMs` widened symmetrically to `targetMs` with the clip's own context, inside the clip. */
|
|
169
|
+
function widenSpeechRange(range: SpeechRange, limits: { minMs: number; targetMs: number; clipMs: number }): SpeechRange {
|
|
170
|
+
const length = range.endMs - range.startMs;
|
|
171
|
+
if (length >= limits.minMs) return range;
|
|
172
|
+
const target = Math.min(limits.targetMs, limits.clipMs);
|
|
173
|
+
const endMs = Math.min(limits.clipMs, Math.max(0, range.startMs - Math.floor((target - length) / 2)) + target);
|
|
174
|
+
const startMs = Math.max(0, endMs - target);
|
|
175
|
+
return withHeard({ clip: range.clip, startMs, endMs }, getHeard(range));
|
|
176
|
+
}
|
|
177
|
+
|
|
73
178
|
/**
|
|
74
|
-
* Where each of one clip's ranges is laid: the whole range, except that two
|
|
75
|
-
* middle of their overlap. Takes ranges of one clip, none inside another (as
|
|
179
|
+
* Where each of one clip's ranges is laid: the part the cut hears (`heard`, else the whole range), except that two
|
|
180
|
+
* overlapping neighbours switch at the middle of their overlap. Takes ranges of one clip, none inside another (as
|
|
181
|
+
* `planSpeechRanges` answers them).
|
|
76
182
|
*/
|
|
77
183
|
export function getSpeechRangeWindows(ranges: readonly SpeechRange[]): SpeechRangeWindow[] {
|
|
78
|
-
const sorted = [...ranges].sort((a, b) => a.startMs - b.startMs);
|
|
184
|
+
const sorted = [...ranges].sort((a, b) => getHeard(a).startMs - getHeard(b).startMs);
|
|
79
185
|
return sorted.map((range, index) => {
|
|
186
|
+
const heard = getHeard(range);
|
|
80
187
|
const before = sorted[index - 1];
|
|
81
188
|
const after = sorted[index + 1];
|
|
82
|
-
const
|
|
83
|
-
const
|
|
189
|
+
const beforeEnd = before === undefined ? null : getHeard(before).endMs;
|
|
190
|
+
const afterStart = after === undefined ? null : getHeard(after).startMs;
|
|
191
|
+
const startMs = beforeEnd !== null && beforeEnd > heard.startMs ? Math.round((heard.startMs + beforeEnd) / 2) : heard.startMs;
|
|
192
|
+
const endMs = afterStart !== null && afterStart < heard.endMs ? Math.round((afterStart + heard.endMs) / 2) : heard.endMs;
|
|
84
193
|
return { range, startMs, endMs };
|
|
85
194
|
});
|
|
86
195
|
}
|
|
@@ -91,11 +200,18 @@ export function getSpeechRangeCache(filmId: string, clipKey: string, range: Spee
|
|
|
91
200
|
return { key, path: getFootageCachePath(options.cacheDir, filmId, "isolated", key) };
|
|
92
201
|
}
|
|
93
202
|
|
|
203
|
+
/** A range's paid cache entry, the app's own: its key (reported on the charge) and file. `clipKey` is the clip's identity. */
|
|
204
|
+
export type SpeechRangeCacheResolver = (range: SpeechRange, clipKey: string) => { key: string; path: string };
|
|
205
|
+
|
|
94
206
|
const seconds = (ms: number): string => (ms / 1000).toFixed(3);
|
|
95
207
|
|
|
96
|
-
/**
|
|
208
|
+
/**
|
|
209
|
+
* One range of a clip's sound (`audio`, read from its start) for the isolation provider: 24-bit PCM for a `.wav`
|
|
210
|
+
* output (lossless, issue #1139), else as `getAudioCodecArgs` encodes it.
|
|
211
|
+
*/
|
|
97
212
|
export function buildRangeAudioArgs(audio: string, range: Pick<SpeechRange, "startMs" | "endMs">, output: string): string[] {
|
|
98
|
-
|
|
213
|
+
const codec = extname(output).toLowerCase() === ".wav" ? ["-c:a", "pcm_s24le"] : getAudioCodecArgs(output);
|
|
214
|
+
return ["-v", "error", "-y", "-ss", seconds(range.startMs), "-t", seconds(range.endMs - range.startMs), "-i", audio, "-vn", ...codec, output];
|
|
99
215
|
}
|
|
100
216
|
|
|
101
217
|
/** A cleaned range laid back: its file, and whether it is mono (panned to both channels at full level). */
|
|
@@ -105,27 +221,81 @@ export interface LaidRange {
|
|
|
105
221
|
isMono: boolean;
|
|
106
222
|
}
|
|
107
223
|
|
|
224
|
+
/** Milliseconds of the linear crossfade at every join between the clip's own sound and a cleaned window. */
|
|
225
|
+
export const DEFAULT_SPEECH_RANGE_FADE_MS = 15;
|
|
226
|
+
|
|
227
|
+
/** Where a window's cleaned sound fades in and out, in milliseconds of the clip: `inStart ≤ inEnd ≤ outStart ≤ outEnd`. */
|
|
228
|
+
export interface SpeechRangeFade {
|
|
229
|
+
inStartMs: number;
|
|
230
|
+
inEndMs: number;
|
|
231
|
+
outStartMs: number;
|
|
232
|
+
outEndMs: number;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* The crossfades of one clip's windows (as `getSpeechRangeWindows` answers them). At a switch between two overlapping
|
|
237
|
+
* neighbours both ranges have sound, so the fade is centred on the switch, widened by half the fade on each side (at
|
|
238
|
+
* most half the overlap) and shared by both windows, whose gains then sum to one. At a range's own edge the range has
|
|
239
|
+
* no sound outside it, so the fade lies inside the window. No fade is longer than half its window.
|
|
240
|
+
*/
|
|
241
|
+
export function getSpeechRangeFades(windows: readonly SpeechRangeWindow[], fadeMs: number): SpeechRangeFade[] {
|
|
242
|
+
const fade = Math.max(0, fadeMs);
|
|
243
|
+
const sorted = [...windows].sort((a, b) => a.startMs - b.startMs);
|
|
244
|
+
const halfWindow = (window: SpeechRangeWindow): number => (window.endMs - window.startMs) / 2;
|
|
245
|
+
/** Half the shared fade at the switch between `before` and `after`; null when they do not overlap. */
|
|
246
|
+
const getSwitchHalf = (before: SpeechRangeWindow | undefined, after: SpeechRangeWindow | undefined): number | null => {
|
|
247
|
+
if (before === undefined || after === undefined || before.range.endMs <= after.range.startMs || before.endMs !== after.startMs) return null;
|
|
248
|
+
const room = Math.min(before.range.endMs - before.endMs, after.startMs - after.range.startMs);
|
|
249
|
+
return Math.floor(Math.min(fade / 2, room, halfWindow(before) / 2, halfWindow(after) / 2));
|
|
250
|
+
};
|
|
251
|
+
return sorted.map((window, index) => {
|
|
252
|
+
const edge = Math.floor(Math.min(fade, halfWindow(window)));
|
|
253
|
+
const startHalf = getSwitchHalf(sorted[index - 1], window);
|
|
254
|
+
const endHalf = getSwitchHalf(window, sorted[index + 1]);
|
|
255
|
+
const [inStartMs, inEndMs] = startHalf === null ? [window.startMs, window.startMs + edge] : [window.startMs - startHalf, window.startMs + startHalf];
|
|
256
|
+
const [outStartMs, outEndMs] = endHalf === null ? [window.endMs - edge, window.endMs] : [window.endMs - endHalf, window.endMs + endHalf];
|
|
257
|
+
return { inStartMs, inEndMs, outStartMs, outEndMs };
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** The gain of a window with `fade` at time `t`, as an ffmpeg expression: 0 outside, 1 inside, linear in the fades. */
|
|
262
|
+
function buildGainExpression(fade: SpeechRangeFade): string {
|
|
263
|
+
const rise = fade.inEndMs > fade.inStartMs ? `clip((t-${seconds(fade.inStartMs)})/${seconds(fade.inEndMs - fade.inStartMs)},0,1)` : `gte(t,${seconds(fade.inStartMs)})`;
|
|
264
|
+
const fall = fade.outEndMs > fade.outStartMs ? `clip((${seconds(fade.outEndMs)}-t)/${seconds(fade.outEndMs - fade.outStartMs)},0,1)` : `lt(t,${seconds(fade.outEndMs)})`;
|
|
265
|
+
return `min(${rise},${fall})`;
|
|
266
|
+
}
|
|
267
|
+
|
|
108
268
|
/**
|
|
109
|
-
* The clip's own sound with each window replaced by its cleaned range
|
|
110
|
-
* window cut from its cleaned file
|
|
111
|
-
*
|
|
269
|
+
* The clip's own sound with each window replaced by its cleaned range, crossfaded at every join (`fadeMs`, default
|
|
270
|
+
* `DEFAULT_SPEECH_RANGE_FADE_MS`; 0 switches at once): each window is cut from its cleaned file to its fade bounds,
|
|
271
|
+
* faded in and out linearly and delayed into place in samples; the base is multiplied by one minus the windows'
|
|
272
|
+
* gains, so the sum keeps its level. As long as the base; 48 kHz stereo like the speech track; a mono answer goes to
|
|
273
|
+
* both channels as it is, not up-mixed at -3 dB.
|
|
112
274
|
*/
|
|
113
|
-
export function buildRangeLayArgs(base: string, laid: readonly LaidRange[], output: string): string[] {
|
|
275
|
+
export function buildRangeLayArgs(base: string, laid: readonly LaidRange[], output: string, options: { fadeMs?: number } = {}): string[] {
|
|
114
276
|
const format = `aformat=sample_rates=${FOOTAGE_SAMPLE_RATE}:channel_layouts=stereo`;
|
|
115
|
-
if (laid.length === 0) return ["-v", "error", "-y", "-i", base, "-vn", "-af", format,
|
|
277
|
+
if (laid.length === 0) return ["-v", "error", "-y", "-i", base, "-vn", "-af", format, ...getAudioCodecArgs(output), output];
|
|
278
|
+
const windows = laid.map(({ window }) => window);
|
|
279
|
+
const fades = getSpeechRangeFades(windows, options.fadeMs ?? DEFAULT_SPEECH_RANGE_FADE_MS);
|
|
280
|
+
const byWindow = new Map([...windows].sort((a, b) => a.startMs - b.startMs).map((window, index) => [window, fades[index]]));
|
|
116
281
|
const inputs = ["-i", base];
|
|
117
|
-
const
|
|
118
|
-
const parts
|
|
282
|
+
const gains: string[] = [];
|
|
283
|
+
const parts: string[] = [];
|
|
119
284
|
laid.forEach(({ window, file, isMono }, index) => {
|
|
285
|
+
const fade = byWindow.get(window);
|
|
286
|
+
if (fade === undefined) return;
|
|
120
287
|
inputs.push("-i", file);
|
|
121
|
-
|
|
122
|
-
const length = window.endMs - window.startMs;
|
|
288
|
+
gains.push(buildGainExpression(fade));
|
|
123
289
|
const channels = isMono ? "pan=stereo|c0=c0|c1=c0," : "";
|
|
290
|
+
const fadeIn = fade.inEndMs > fade.inStartMs ? `,afade=t=in:st=0:d=${seconds(fade.inEndMs - fade.inStartMs)}` : "";
|
|
291
|
+
const fadeOut = fade.outEndMs > fade.outStartMs ? `,afade=t=out:st=${seconds(fade.outStartMs - fade.inStartMs)}:d=${seconds(fade.outEndMs - fade.outStartMs)}` : "";
|
|
292
|
+
const delay = Math.round((fade.inStartMs * FOOTAGE_SAMPLE_RATE) / 1000);
|
|
124
293
|
parts.push(
|
|
125
|
-
`[${index + 1}:a]atrim=start=${seconds(
|
|
294
|
+
`[${index + 1}:a]atrim=start=${seconds(fade.inStartMs - window.range.startMs)}:end=${seconds(fade.outEndMs - window.range.startMs)},asetpts=PTS-STARTPTS,${channels}${format}${fadeIn}${fadeOut},adelay=${String(delay)}S:all=1[w${String(index)}]`,
|
|
126
295
|
);
|
|
127
296
|
});
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
297
|
+
parts.unshift(`[0:a]${format},aeval='val(ch)*(1-(${gains.join("+")}))':c=same[base]`);
|
|
298
|
+
const mixed = laid.map((_, index) => `[w${String(index)}]`).join("");
|
|
299
|
+
parts.push(`[base]${mixed}amix=inputs=${String(laid.length + 1)}:duration=first:normalize=0[speech]`);
|
|
300
|
+
return ["-v", "error", "-y", ...inputs, "-filter_complex", parts.join(";"), "-map", "[speech]", ...getAudioCodecArgs(output), output];
|
|
131
301
|
}
|
package/src/index.ts
CHANGED
|
@@ -84,6 +84,17 @@ export {
|
|
|
84
84
|
type TtsRequest,
|
|
85
85
|
} from "./voice/voiceover.js";
|
|
86
86
|
export type { TtsEstimate, TtsInput, TtsProvider, TtsRecording, TtsResult } from "./voice/provider.js";
|
|
87
|
+
export {
|
|
88
|
+
describeProviderFailure,
|
|
89
|
+
getConnectionFailure,
|
|
90
|
+
getLocalRefusal,
|
|
91
|
+
getProviderFailure,
|
|
92
|
+
getProviderFailureKind,
|
|
93
|
+
readProviderErrorBody,
|
|
94
|
+
PROVIDER_MESSAGE_LENGTH,
|
|
95
|
+
type ProviderFailure,
|
|
96
|
+
type ProviderFailureKind,
|
|
97
|
+
} from "./voice/provider-failure.js";
|
|
87
98
|
export { ELEVENLABS_API_KEY_ENV, ELEVENLABS_PROVIDER_ID, createElevenLabsProvider, type ElevenLabsOptions } from "./voice/elevenlabs.js";
|
|
88
99
|
export { createFakeTtsProvider, type FakeTtsOptions, type FakeTtsProvider } from "./voice/fake.js";
|
|
89
100
|
export { createTtsProvider, type CreateTtsProviderOptions, type TtsProviderId } from "./voice/providers.js";
|
|
@@ -156,11 +167,14 @@ export {
|
|
|
156
167
|
loadOgFonts,
|
|
157
168
|
renderConfiguredOgImage,
|
|
158
169
|
renderOgImage,
|
|
170
|
+
renderPhotoCover,
|
|
159
171
|
type ConfiguredOgImageOptions,
|
|
160
172
|
type OgBrand,
|
|
161
173
|
type OgImageInput,
|
|
162
174
|
type OgResult,
|
|
163
175
|
type OgTemplateId,
|
|
176
|
+
type PhotoCover,
|
|
177
|
+
type PhotoCoverInput,
|
|
164
178
|
} from "./og/index.js";
|
|
165
179
|
export { buildPosts, postsMarkdown, type PlatformPost, type PostsInput } from "./posts/posts.js";
|
|
166
180
|
export { createActionScene, getLocator, runAction, type LocatorSource } from "./record/actions.js";
|
|
@@ -293,6 +307,7 @@ export {
|
|
|
293
307
|
type SttTranscript,
|
|
294
308
|
} from "./footage/providers.js";
|
|
295
309
|
export {
|
|
310
|
+
ELEVENLABS_ISOLATION_MIN_SECONDS,
|
|
296
311
|
ELEVENLABS_MUSIC_MODEL,
|
|
297
312
|
ELEVENLABS_STT_MODEL,
|
|
298
313
|
createElevenLabsFootageProviders,
|
|
@@ -309,6 +324,7 @@ export {
|
|
|
309
324
|
createFakeMusicProvider,
|
|
310
325
|
createFakeSttProvider,
|
|
311
326
|
makePlaceholderTranscript,
|
|
327
|
+
type FakeIsolationOptions,
|
|
312
328
|
type FakeIsolationProvider,
|
|
313
329
|
type FakeMusicProvider,
|
|
314
330
|
type FakeSttOptions,
|
|
@@ -419,6 +435,18 @@ export {
|
|
|
419
435
|
type PicturePieceArgsInput,
|
|
420
436
|
} from "./footage/picture.js";
|
|
421
437
|
export {
|
|
438
|
+
SPEECH_PRE_GAIN,
|
|
439
|
+
buildClipSoundArgs,
|
|
440
|
+
buildClipSoundMeasureArgs,
|
|
441
|
+
getAudioCodecArgs,
|
|
442
|
+
getClipSoundPath,
|
|
443
|
+
getPreGainDb,
|
|
444
|
+
prepareClipSounds,
|
|
445
|
+
type ClipSound,
|
|
446
|
+
type ClipSoundInput,
|
|
447
|
+
} from "./footage/clip-sound.js";
|
|
448
|
+
export {
|
|
449
|
+
AUDIO_BITRATE_RANGE_KBPS,
|
|
422
450
|
DEFAULT_LOUDNESS,
|
|
423
451
|
buildLoudnessArgs,
|
|
424
452
|
buildLoudnessMeasureArgs,
|
|
@@ -501,18 +529,43 @@ export {
|
|
|
501
529
|
type FootageEstimateItem,
|
|
502
530
|
} from "./footage/paid.js";
|
|
503
531
|
export {
|
|
532
|
+
DEFAULT_SPEECH_RANGE_FADE_MS,
|
|
504
533
|
DEFAULT_SPEECH_RANGE_MARGIN,
|
|
534
|
+
DEFAULT_SPEECH_RANGE_MERGE_GAP,
|
|
535
|
+
DEFAULT_SPEECH_RANGE_MIN_PAD,
|
|
505
536
|
buildRangeAudioArgs,
|
|
506
537
|
buildRangeLayArgs,
|
|
507
538
|
getSpeechRangeCache,
|
|
539
|
+
getSpeechRangeFades,
|
|
508
540
|
getSpeechRangeWindows,
|
|
509
541
|
planSpeechRanges,
|
|
510
542
|
type LaidRange,
|
|
543
|
+
type PlanSpeechRangesOptions,
|
|
511
544
|
type SpeechRange,
|
|
545
|
+
type SpeechRangeCacheResolver,
|
|
512
546
|
type SpeechRangeClip,
|
|
547
|
+
type SpeechRangeFade,
|
|
548
|
+
type SpeechRangeSkip,
|
|
513
549
|
type SpeechRangeWindow,
|
|
514
550
|
} from "./footage/speech-ranges.js";
|
|
515
|
-
export {
|
|
551
|
+
export {
|
|
552
|
+
DEFAULT_SPEECH_RANGE_ATTEMPTS,
|
|
553
|
+
DEFAULT_SPEECH_RANGE_RETRY_DELAYS_MS,
|
|
554
|
+
isolateSpeechRanges,
|
|
555
|
+
type IsolateSpeechRangesOptions,
|
|
556
|
+
type IsolatedSpeech,
|
|
557
|
+
type IsolatedSpeechRange,
|
|
558
|
+
type SpeechRangeOutcome,
|
|
559
|
+
type SpeechRangeRetry,
|
|
560
|
+
} from "./footage/isolate-ranges.js";
|
|
561
|
+
export {
|
|
562
|
+
HYPERFRAMES_CAPTURE_STAGES,
|
|
563
|
+
readHyperframesCapture,
|
|
564
|
+
type HyperframesCapture,
|
|
565
|
+
type HyperframesCapturePath,
|
|
566
|
+
type HyperframesCaptureStage,
|
|
567
|
+
type HyperframesGpuMode,
|
|
568
|
+
} from "./footage/hyperframes-capture.js";
|
|
516
569
|
export { RENDER_MIN_FREE_BYTES, findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./footage/disk.js";
|
|
517
570
|
export { DRAFT_FILTERS, REQUIRED_DECODERS, REQUIRED_FILTERS, checkFootageTools, findBrowserProblem, readFfmpegListing, type FootageToolsOptions } from "./footage/tools.js";
|
|
518
571
|
export {
|
|
@@ -546,6 +599,7 @@ export {
|
|
|
546
599
|
type RenderedFootage,
|
|
547
600
|
type TranscribeOutcome,
|
|
548
601
|
} from "./footage/pipeline.js";
|
|
602
|
+
export { getProviderFootageFailure, isRetryableFootageFailure, type FootageError, type FootageFailure, type FootageFailureKind } from "./footage/failure.js";
|
|
549
603
|
export {
|
|
550
604
|
formatMessage,
|
|
551
605
|
getMarketingMessages,
|
package/src/og/index.ts
CHANGED
|
@@ -18,6 +18,29 @@ export {
|
|
|
18
18
|
type OgBrand,
|
|
19
19
|
type OgImageInput,
|
|
20
20
|
} from "./render.js";
|
|
21
|
+
export {
|
|
22
|
+
MAX_COVER_CAPTION_LENGTH,
|
|
23
|
+
PHOTO_COVER_QUALITY,
|
|
24
|
+
PHOTO_COVER_SIZE,
|
|
25
|
+
getCoverBand,
|
|
26
|
+
getCoverCaptionSize,
|
|
27
|
+
renderPhotoCover,
|
|
28
|
+
type CoverBand,
|
|
29
|
+
type PhotoCover,
|
|
30
|
+
type PhotoCoverInput,
|
|
31
|
+
} from "./photo-cover.js";
|
|
32
|
+
export {
|
|
33
|
+
getCoverCrop,
|
|
34
|
+
getOrientationMatrix,
|
|
35
|
+
getOrientedSize,
|
|
36
|
+
readPhotoInfo,
|
|
37
|
+
readTiffOrientation,
|
|
38
|
+
type CoverCrop,
|
|
39
|
+
type FocusPoint,
|
|
40
|
+
type PhotoFormat,
|
|
41
|
+
type PhotoInfo,
|
|
42
|
+
type PhotoOrientation,
|
|
43
|
+
} from "./photo.js";
|
|
21
44
|
export { OG_TEMPLATE_IDS, OG_TEMPLATES, isOgTemplateId, type OgTemplateId } from "./templates/index.js";
|
|
22
45
|
export { PORTRAIT_BASE, getLayoutScale, type OgLayout, type OgTemplateContext } from "./templates/context.js";
|
|
23
46
|
export {
|