@softure-ai/marketing-kit 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +108 -3
- package/dist/cli/footage.d.ts +6 -0
- package/dist/cli/footage.d.ts.map +1 -0
- package/dist/cli/footage.js +95 -0
- package/dist/cli/footage.js.map +1 -0
- package/dist/cli/main.js +18 -1
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/options.d.ts +18 -3
- package/dist/cli/options.d.ts.map +1 -1
- package/dist/cli/options.js +41 -2
- package/dist/cli/options.js.map +1 -1
- package/dist/compose/compose.d.ts.map +1 -1
- package/dist/compose/compose.js +1 -22
- package/dist/compose/compose.js.map +1 -1
- package/dist/compose/html.d.ts +6 -0
- package/dist/compose/html.d.ts.map +1 -0
- package/dist/compose/html.js +24 -0
- package/dist/compose/html.js.map +1 -0
- package/dist/config/actions-schema.d.ts +14 -14
- package/dist/config/config.d.ts +16 -0
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +88 -0
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +133 -70
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +110 -3
- package/dist/config/schema.js.map +1 -1
- package/dist/config/shot-steps.d.ts +16 -16
- package/dist/footage/cache.d.ts +23 -0
- package/dist/footage/cache.d.ts.map +1 -0
- package/dist/footage/cache.js +51 -0
- package/dist/footage/cache.js.map +1 -0
- package/dist/footage/compose.d.ts +50 -0
- package/dist/footage/compose.d.ts.map +1 -0
- package/dist/footage/compose.js +166 -0
- package/dist/footage/compose.js.map +1 -0
- package/dist/footage/edl.d.ts +83 -0
- package/dist/footage/edl.d.ts.map +1 -0
- package/dist/footage/edl.js +166 -0
- package/dist/footage/edl.js.map +1 -0
- package/dist/footage/elevenlabs.d.ts +31 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -0
- package/dist/footage/elevenlabs.js +140 -0
- package/dist/footage/elevenlabs.js.map +1 -0
- package/dist/footage/fake.d.ts +38 -0
- package/dist/footage/fake.d.ts.map +1 -0
- package/dist/footage/fake.js +66 -0
- package/dist/footage/fake.js.map +1 -0
- package/dist/footage/film.d.ts +103 -0
- package/dist/footage/film.d.ts.map +1 -0
- package/dist/footage/film.js +29 -0
- package/dist/footage/film.js.map +1 -0
- package/dist/footage/music.d.ts +37 -0
- package/dist/footage/music.d.ts.map +1 -0
- package/dist/footage/music.js +63 -0
- package/dist/footage/music.js.map +1 -0
- package/dist/footage/normalize.d.ts +41 -0
- package/dist/footage/normalize.d.ts.map +1 -0
- package/dist/footage/normalize.js +114 -0
- package/dist/footage/normalize.js.map +1 -0
- package/dist/footage/pipeline.d.ts +105 -0
- package/dist/footage/pipeline.d.ts.map +1 -0
- package/dist/footage/pipeline.js +385 -0
- package/dist/footage/pipeline.js.map +1 -0
- package/dist/footage/probe.d.ts +64 -0
- package/dist/footage/probe.d.ts.map +1 -0
- package/dist/footage/probe.js +138 -0
- package/dist/footage/probe.js.map +1 -0
- package/dist/footage/providers.d.ts +61 -0
- package/dist/footage/providers.d.ts.map +1 -0
- package/dist/footage/providers.js +13 -0
- package/dist/footage/providers.js.map +1 -0
- package/dist/footage/render.d.ts +20 -0
- package/dist/footage/render.d.ts.map +1 -0
- package/dist/footage/render.js +76 -0
- package/dist/footage/render.js.map +1 -0
- package/dist/index.d.ts +13 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -2
- package/dist/index.js.map +1 -1
- package/dist/render/render.d.ts +6 -0
- package/dist/render/render.d.ts.map +1 -1
- package/dist/render/render.js +1 -1
- package/dist/render/render.js.map +1 -1
- package/package.json +1 -1
- package/schema/marketing.schema.json +254 -2
- package/src/cli/footage.ts +87 -0
- package/src/cli/main.ts +17 -1
- package/src/cli/options.ts +51 -4
- package/src/compose/compose.ts +1 -25
- package/src/compose/html.ts +28 -0
- package/src/config/config.ts +97 -0
- package/src/config/schema.ts +114 -4
- package/src/footage/cache.ts +66 -0
- package/src/footage/compose.ts +213 -0
- package/src/footage/edl.ts +204 -0
- package/src/footage/elevenlabs.ts +154 -0
- package/src/footage/fake.ts +92 -0
- package/src/footage/film.ts +111 -0
- package/src/footage/music.ts +86 -0
- package/src/footage/normalize.ts +133 -0
- package/src/footage/pipeline.ts +447 -0
- package/src/footage/probe.ts +172 -0
- package/src/footage/providers.ts +71 -0
- package/src/footage/render.ts +82 -0
- package/src/index.ts +109 -1
- package/src/render/render.ts +1 -1
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
|
|
3
|
+
import type { TimedWord } from "../voice/voiceover.js";
|
|
4
|
+
import { CUT_TRANSITIONS, type ClipRole, type CutOptions, type CutTransition, type FootageTransitions } from "./film.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The edit decision list: which parts of which clips the film keeps, in order, and how each part starts. Built from
|
|
8
|
+
* the transcript (`buildEdl`), written as plain JSON next to the build so a person or an agent can check or change it
|
|
9
|
+
* before the render, and read back through `edlSchema`.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export const SEGMENT_MOTIONS = ["none", "kenburns"] as const;
|
|
13
|
+
|
|
14
|
+
export type SegmentMotion = (typeof SEGMENT_MOTIONS)[number];
|
|
15
|
+
|
|
16
|
+
export interface EdlSegment {
|
|
17
|
+
/** The clip's id. */
|
|
18
|
+
clip: string;
|
|
19
|
+
/** Seconds in the trimmed clip. */
|
|
20
|
+
start: number;
|
|
21
|
+
end: number;
|
|
22
|
+
transition: CutTransition;
|
|
23
|
+
/** `kenburns`: a slow zoom over a segment without speech. */
|
|
24
|
+
motion: SegmentMotion;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface Edl {
|
|
28
|
+
version: 1;
|
|
29
|
+
film: string;
|
|
30
|
+
segments: EdlSegment[];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export const edlSchema = z.strictObject({
|
|
34
|
+
version: z.literal(1),
|
|
35
|
+
film: z.string().min(1),
|
|
36
|
+
segments: z
|
|
37
|
+
.array(
|
|
38
|
+
z.strictObject({
|
|
39
|
+
clip: z.string().min(1),
|
|
40
|
+
start: z.number().min(0),
|
|
41
|
+
end: z.number().positive(),
|
|
42
|
+
transition: z.enum(CUT_TRANSITIONS),
|
|
43
|
+
motion: z.enum(SEGMENT_MOTIONS),
|
|
44
|
+
}),
|
|
45
|
+
)
|
|
46
|
+
.min(1, "the cut needs at least one segment"),
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
/** A clip as the cut sees it: its role, its trimmed length and its transcript (times from the trimmed start). */
|
|
50
|
+
export interface EdlClip {
|
|
51
|
+
id: string;
|
|
52
|
+
role: ClipRole;
|
|
53
|
+
duration: number;
|
|
54
|
+
words: TimedWord[];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Shortest segment kept, in seconds: anything shorter reads as a glitch. */
|
|
58
|
+
export const MIN_SEGMENT_SECONDS = 0.25;
|
|
59
|
+
|
|
60
|
+
const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
|
|
61
|
+
|
|
62
|
+
/** A word reduced for comparison: lower case, letters and digits only, repeated letters collapsed (`Yyyy,` → `y`). */
|
|
63
|
+
export function normalizeWord(text: string): string {
|
|
64
|
+
return text
|
|
65
|
+
.toLowerCase()
|
|
66
|
+
.replace(/[^\p{L}\p{N}]/gu, "")
|
|
67
|
+
.replace(/(.)\1+/gu, "$1");
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Whether the word is one of the fillers (compared as `normalizeWord` writes both). */
|
|
71
|
+
export function isFiller(text: string, fillers: readonly string[]): boolean {
|
|
72
|
+
const word = normalizeWord(text);
|
|
73
|
+
return word.length > 0 && fillers.some((filler) => normalizeWord(filler) === word);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
interface Span {
|
|
77
|
+
start: number;
|
|
78
|
+
end: number;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** The kept spans of a talking clip: runs of spoken words split at fillers and long pauses, padded, never into a neighbour word. */
|
|
82
|
+
function getTalkSpans(clip: EdlClip, options: CutOptions): Span[] {
|
|
83
|
+
const runs: TimedWord[][] = [];
|
|
84
|
+
let run: TimedWord[] = [];
|
|
85
|
+
const words = [...clip.words].sort((a, b) => a.start - b.start);
|
|
86
|
+
words.forEach((word, index) => {
|
|
87
|
+
const previous = words[index - 1];
|
|
88
|
+
if (isFiller(word.text, options.fillers)) {
|
|
89
|
+
if (run.length > 0) runs.push(run);
|
|
90
|
+
run = [];
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
if (run.length > 0 && previous !== undefined && word.start - previous.end > options.pauseSeconds) {
|
|
94
|
+
runs.push(run);
|
|
95
|
+
run = [];
|
|
96
|
+
}
|
|
97
|
+
run.push(word);
|
|
98
|
+
});
|
|
99
|
+
if (run.length > 0) runs.push(run);
|
|
100
|
+
const spans: Span[] = [];
|
|
101
|
+
for (const kept of runs) {
|
|
102
|
+
const first = kept[0] as TimedWord;
|
|
103
|
+
const last = kept[kept.length - 1] as TimedWord;
|
|
104
|
+
const before = words[words.indexOf(first) - 1];
|
|
105
|
+
const after = words[words.indexOf(last) + 1];
|
|
106
|
+
const start = Math.max(0, first.start - options.padSeconds, before?.end ?? 0);
|
|
107
|
+
const end = Math.min(clip.duration, last.end + options.padSeconds, after?.start ?? clip.duration);
|
|
108
|
+
const previous = spans.at(-1);
|
|
109
|
+
if (previous !== undefined && start <= previous.end + 0.01) previous.end = Math.max(previous.end, end);
|
|
110
|
+
else spans.push({ start, end });
|
|
111
|
+
}
|
|
112
|
+
return spans.filter((span) => span.end - span.start >= MIN_SEGMENT_SECONDS);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Cuts the segments to `maxSeconds`: the last one that crosses it is shortened, later ones dropped. */
|
|
116
|
+
function limitLength(segments: EdlSegment[], maxSeconds: number | null): EdlSegment[] {
|
|
117
|
+
if (maxSeconds === null) return segments;
|
|
118
|
+
const kept: EdlSegment[] = [];
|
|
119
|
+
let total = 0;
|
|
120
|
+
for (const segment of segments) {
|
|
121
|
+
const left = maxSeconds - total;
|
|
122
|
+
if (left < MIN_SEGMENT_SECONDS) break;
|
|
123
|
+
const length = segment.end - segment.start;
|
|
124
|
+
kept.push(length <= left ? segment : { ...segment, end: r3(segment.start + left) });
|
|
125
|
+
total += Math.min(length, left);
|
|
126
|
+
}
|
|
127
|
+
return kept;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* The cut from the transcripts. A talking clip keeps its runs of words: fillers are cut out, a pause longer than
|
|
132
|
+
* `pauseSeconds` is cut, `padSeconds` stays around each cut. A talking clip without words, and every b-roll clip, is
|
|
133
|
+
* kept whole (b-roll with a slow zoom). The first segment starts with a cut; later ones with `transitions.jump`
|
|
134
|
+
* inside one clip and `transitions.clip` between clips.
|
|
135
|
+
*/
|
|
136
|
+
export function buildEdl(filmId: string, clips: EdlClip[], options: CutOptions, transitions: FootageTransitions): Edl {
|
|
137
|
+
const segments: EdlSegment[] = [];
|
|
138
|
+
for (const clip of clips) {
|
|
139
|
+
const spans = clip.role === "talk" ? getTalkSpans(clip, options) : [];
|
|
140
|
+
const kept = spans.length > 0 ? spans : [{ start: 0, end: clip.duration }];
|
|
141
|
+
for (const span of kept) {
|
|
142
|
+
const previous = segments.at(-1);
|
|
143
|
+
const transition = previous === undefined ? "cut" : previous.clip === clip.id ? transitions.jump : transitions.clip;
|
|
144
|
+
segments.push({ clip: clip.id, start: r3(span.start), end: r3(span.end), transition, motion: clip.role === "broll" ? "kenburns" : "none" });
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return { version: 1, film: filmId, segments: limitLength(segments, options.maxSeconds) };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** What makes an EDL (a hand-edited one above all) unusable with these clips; empty when it is fine. */
|
|
151
|
+
export function findEdlProblems(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[], filmId: string): string[] {
|
|
152
|
+
const problems: string[] = [];
|
|
153
|
+
if (edl.film !== filmId) problems.push(`it belongs to film "${edl.film}", not "${filmId}"`);
|
|
154
|
+
const durations = new Map(clips.map((clip) => [clip.id, clip.duration]));
|
|
155
|
+
edl.segments.forEach((segment, index) => {
|
|
156
|
+
const duration = durations.get(segment.clip);
|
|
157
|
+
const at = `segments[${index}]`;
|
|
158
|
+
if (duration === undefined) problems.push(`${at}.clip: no clip "${segment.clip}" in the film`);
|
|
159
|
+
else if (segment.end > duration + 0.05) problems.push(`${at}.end: ${segment.end} s is past the clip's end (${duration.toFixed(2)} s)`);
|
|
160
|
+
if (segment.end - segment.start < MIN_SEGMENT_SECONDS) problems.push(`${at}: shorter than ${MIN_SEGMENT_SECONDS} s (start ${segment.start}, end ${segment.end})`);
|
|
161
|
+
});
|
|
162
|
+
return problems;
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/** Reads an EDL file's JSON; an error names the first problems by path. */
|
|
166
|
+
export function readEdl(json: unknown): { ok: true; edl: Edl } | { ok: false; error: string } {
|
|
167
|
+
const parsed = edlSchema.safeParse(json);
|
|
168
|
+
if (parsed.success) return { ok: true, edl: parsed.data };
|
|
169
|
+
const issues = parsed.error.issues.slice(0, 5).map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`);
|
|
170
|
+
return { ok: false, error: issues.join("; ") };
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export function getEdlDuration(edl: Edl): number {
|
|
174
|
+
return r3(edl.segments.reduce((sum, segment) => sum + (segment.end - segment.start), 0));
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Where each segment starts in the film, in seconds. */
|
|
178
|
+
export function getSegmentStarts(edl: Edl): number[] {
|
|
179
|
+
const starts: number[] = [];
|
|
180
|
+
let time = 0;
|
|
181
|
+
for (const segment of edl.segments) {
|
|
182
|
+
starts.push(r3(time));
|
|
183
|
+
time += segment.end - segment.start;
|
|
184
|
+
}
|
|
185
|
+
return starts;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* The words the film says, in film time: each clip's words that fall in a kept segment (by their middle), shifted by
|
|
190
|
+
* where the segment lands; fillers left out. Every cut shifts every later word.
|
|
191
|
+
*/
|
|
192
|
+
export function remapWords(edl: Edl, words: Record<string, TimedWord[]>, fillers: readonly string[]): TimedWord[] {
|
|
193
|
+
const starts = getSegmentStarts(edl);
|
|
194
|
+
const film: TimedWord[] = [];
|
|
195
|
+
edl.segments.forEach((segment, index) => {
|
|
196
|
+
const offset = (starts[index] as number) - segment.start;
|
|
197
|
+
for (const word of words[segment.clip] ?? []) {
|
|
198
|
+
const middle = (word.start + word.end) / 2;
|
|
199
|
+
if (middle < segment.start || middle >= segment.end || isFiller(word.text, fillers)) continue;
|
|
200
|
+
film.push({ text: word.text, start: r3(Math.max(word.start, segment.start) + offset), end: r3(Math.min(word.end, segment.end) + offset) });
|
|
201
|
+
}
|
|
202
|
+
});
|
|
203
|
+
return film;
|
|
204
|
+
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { basename } from "node:path";
|
|
3
|
+
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
|
|
6
|
+
import { ELEVENLABS_API_KEY_ENV } from "../voice/elevenlabs.js";
|
|
7
|
+
import { err, ok, type TtsResult } from "../voice/provider.js";
|
|
8
|
+
import { ELEVENLABS_API_URL, type TimedWord } from "../voice/voiceover.js";
|
|
9
|
+
import type { IsolationProvider, MusicProvider, SttProvider, SttTranscript } from "./providers.js";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* ElevenLabs behind the footage providers (API reference checked 2026-10-29): Scribe speech to text
|
|
13
|
+
* (`/v1/speech-to-text`, word timestamps), Voice Isolator (`/v1/audio-isolation`), Music (`/v1/music`) and Video to
|
|
14
|
+
* Music (`/v1/music/video-to-music`). Each takes an injected `fetch`, so tests reach no network.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
export const ELEVENLABS_STT_MODEL = "scribe_v2";
|
|
18
|
+
export const ELEVENLABS_MUSIC_MODEL = "music_v1";
|
|
19
|
+
const ERROR_DETAIL_LENGTH = 300;
|
|
20
|
+
/** ElevenLabs Music takes 3 s to 10 min. */
|
|
21
|
+
const MUSIC_MIN_MS = 3000;
|
|
22
|
+
const MUSIC_MAX_MS = 600_000;
|
|
23
|
+
|
|
24
|
+
export interface ElevenLabsFootageOptions {
|
|
25
|
+
/** The API key; a missing one fails the call, never the estimate. */
|
|
26
|
+
apiKey: string | undefined;
|
|
27
|
+
fetch?: typeof globalThis.fetch;
|
|
28
|
+
/** Scribe's model; `scribe_v2` by default. */
|
|
29
|
+
sttModel?: string;
|
|
30
|
+
musicModel?: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
const scribeSchema = z.looseObject({
|
|
34
|
+
language_code: z.string().optional(),
|
|
35
|
+
words: z.array(z.looseObject({ text: z.string(), start: z.number().nullish(), end: z.number().nullish(), type: z.string().optional() })),
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const round = (value: number): number => Math.round(value * 1000) / 1000 || 0;
|
|
39
|
+
|
|
40
|
+
/** Scribe's response read as timed words: entries of type `word` only (no spacing, no sound events). */
|
|
41
|
+
export function readScribeResponse(body: unknown, language: string): TtsResult<SttTranscript> {
|
|
42
|
+
const parsed = scribeSchema.safeParse(body);
|
|
43
|
+
if (!parsed.success) return err("ElevenLabs speech to text: the response has no words list.");
|
|
44
|
+
const words: TimedWord[] = [];
|
|
45
|
+
for (const word of parsed.data.words) {
|
|
46
|
+
if ((word.type ?? "word") !== "word") continue;
|
|
47
|
+
const text = word.text.trim();
|
|
48
|
+
if (text.length === 0 || word.start == null || word.end == null) continue;
|
|
49
|
+
words.push({ text, start: round(word.start), end: round(word.end) });
|
|
50
|
+
}
|
|
51
|
+
return ok({ words, language: parsed.data.language_code ?? language });
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function fileBlob(path: string): Blob {
|
|
55
|
+
return new Blob([readFileSync(path)]);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function send(options: ElevenLabsFootageOptions, label: string, path: string, init: RequestInit): Promise<TtsResult<Response>> {
|
|
59
|
+
if (options.apiKey === undefined || options.apiKey.length === 0) return err(`no ${ELEVENLABS_API_KEY_ENV} in the environment.`);
|
|
60
|
+
const request = options.fetch ?? globalThis.fetch;
|
|
61
|
+
let response: Response;
|
|
62
|
+
try {
|
|
63
|
+
response = await request(`${ELEVENLABS_API_URL}${path}`, { ...init, headers: { ...init.headers, "xi-api-key": options.apiKey } });
|
|
64
|
+
} catch (error) {
|
|
65
|
+
return err(`ElevenLabs ${label}: the connection failed (${String(error)}).`);
|
|
66
|
+
}
|
|
67
|
+
if (!response.ok) {
|
|
68
|
+
const detail = await response.text().catch((error: unknown) => `(unreadable body: ${String(error)})`);
|
|
69
|
+
return err(`ElevenLabs ${label} answered ${response.status}: ${detail.slice(0, ERROR_DETAIL_LENGTH)}`);
|
|
70
|
+
}
|
|
71
|
+
return ok(response);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
async function readAudio(label: string, response: Response): Promise<TtsResult<Buffer>> {
|
|
75
|
+
try {
|
|
76
|
+
const audio = Buffer.from(await response.arrayBuffer());
|
|
77
|
+
return audio.length > 0 ? ok(audio) : err(`ElevenLabs ${label}: the response has no audio.`);
|
|
78
|
+
} catch (error) {
|
|
79
|
+
return err(`ElevenLabs ${label}: reading the audio failed (${String(error)}).`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function createElevenLabsSttProvider(options: ElevenLabsFootageOptions): SttProvider {
|
|
84
|
+
const model = options.sttModel ?? ELEVENLABS_STT_MODEL;
|
|
85
|
+
return {
|
|
86
|
+
id: "elevenlabs",
|
|
87
|
+
model,
|
|
88
|
+
async transcribe(input) {
|
|
89
|
+
const form = new FormData();
|
|
90
|
+
form.append("model_id", model);
|
|
91
|
+
form.append("file", fileBlob(input.audio), basename(input.audio));
|
|
92
|
+
form.append("language_code", input.language);
|
|
93
|
+
form.append("timestamps_granularity", "word");
|
|
94
|
+
form.append("tag_audio_events", "false");
|
|
95
|
+
const response = await send(options, "speech to text", "/v1/speech-to-text", { method: "POST", body: form });
|
|
96
|
+
if (!response.ok) return response;
|
|
97
|
+
let body: unknown;
|
|
98
|
+
try {
|
|
99
|
+
body = (await response.value.json()) as unknown;
|
|
100
|
+
} catch (error) {
|
|
101
|
+
return err(`ElevenLabs speech to text: the response is not JSON (${String(error)}).`);
|
|
102
|
+
}
|
|
103
|
+
return readScribeResponse(body, input.language);
|
|
104
|
+
},
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export function createElevenLabsIsolationProvider(options: ElevenLabsFootageOptions): IsolationProvider {
|
|
109
|
+
return {
|
|
110
|
+
id: "elevenlabs",
|
|
111
|
+
async isolate(input) {
|
|
112
|
+
const form = new FormData();
|
|
113
|
+
form.append("audio", fileBlob(input.audio), basename(input.audio));
|
|
114
|
+
const response = await send(options, "voice isolation", "/v1/audio-isolation", { method: "POST", body: form });
|
|
115
|
+
return response.ok ? readAudio("voice isolation", response.value) : response;
|
|
116
|
+
},
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** The length ElevenLabs Music is asked for: the film's, within the service's 3 s to 10 min. */
|
|
121
|
+
export function getMusicLengthMs(seconds: number): number {
|
|
122
|
+
return Math.min(MUSIC_MAX_MS, Math.max(MUSIC_MIN_MS, Math.ceil(seconds * 1000)));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export function createElevenLabsMusicProvider(options: ElevenLabsFootageOptions): MusicProvider {
|
|
126
|
+
const model = options.musicModel ?? ELEVENLABS_MUSIC_MODEL;
|
|
127
|
+
return {
|
|
128
|
+
id: "elevenlabs",
|
|
129
|
+
model,
|
|
130
|
+
async compose(input) {
|
|
131
|
+
const body = { prompt: input.prompt, music_length_ms: getMusicLengthMs(input.seconds), model_id: model, force_instrumental: true };
|
|
132
|
+
const response = await send(options, "music", "/v1/music?output_format=mp3_44100_128", {
|
|
133
|
+
method: "POST",
|
|
134
|
+
headers: { "Content-Type": "application/json" },
|
|
135
|
+
body: JSON.stringify(body),
|
|
136
|
+
});
|
|
137
|
+
return response.ok ? readAudio("music", response.value) : response;
|
|
138
|
+
},
|
|
139
|
+
async scoreVideo(input) {
|
|
140
|
+
const form = new FormData();
|
|
141
|
+
form.append("videos", fileBlob(input.video), basename(input.video));
|
|
142
|
+
if (input.description.length > 0) form.append("description", input.description);
|
|
143
|
+
for (const tag of input.tags) form.append("tags", tag);
|
|
144
|
+
const response = await send(options, "video to music", "/v1/music/video-to-music?output_format=mp3_44100_128", { method: "POST", body: form });
|
|
145
|
+
return response.ok ? readAudio("video to music", response.value) : response;
|
|
146
|
+
},
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** The ElevenLabs footage providers with the key from the environment (read, never logged). */
|
|
151
|
+
export function createElevenLabsFootageProviders(env: Record<string, string | undefined>, fetch?: typeof globalThis.fetch) {
|
|
152
|
+
const options: ElevenLabsFootageOptions = { apiKey: env[ELEVENLABS_API_KEY_ENV], ...(fetch === undefined ? {} : { fetch }) };
|
|
153
|
+
return { stt: createElevenLabsSttProvider(options), isolation: createElevenLabsIsolationProvider(options), music: createElevenLabsMusicProvider(options) };
|
|
154
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { basename } from "node:path";
|
|
3
|
+
|
|
4
|
+
import { ok, type TtsResult } from "../voice/provider.js";
|
|
5
|
+
import type { TimedWord } from "../voice/voiceover.js";
|
|
6
|
+
import type { IsolationProvider, MusicProvider, SttInput, SttProvider, SttTranscript } from "./providers.js";
|
|
7
|
+
|
|
8
|
+
/** Providers that spend nothing and reach no network, for tests of the footage pipeline and of a project's own flows. */
|
|
9
|
+
|
|
10
|
+
export interface FakeSttOptions {
|
|
11
|
+
/** Words per audio file, by the file's name up to its first dot (the pipeline names a clip's audio `<clip id>.*`). */
|
|
12
|
+
transcripts?: Record<string, TimedWord[]>;
|
|
13
|
+
/** Makes every call fail with this message. */
|
|
14
|
+
failWith?: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface FakeSttProvider extends SttProvider {
|
|
18
|
+
readonly calls: SttInput[];
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function createFakeSttProvider(options: FakeSttOptions = {}): FakeSttProvider {
|
|
22
|
+
const calls: SttInput[] = [];
|
|
23
|
+
return {
|
|
24
|
+
id: "fake",
|
|
25
|
+
model: "fake-stt",
|
|
26
|
+
calls,
|
|
27
|
+
transcribe(input): Promise<TtsResult<SttTranscript>> {
|
|
28
|
+
calls.push(input);
|
|
29
|
+
if (options.failWith !== undefined) return Promise.resolve({ ok: false, error: options.failWith });
|
|
30
|
+
const name = basename(input.audio).split(".")[0] ?? "";
|
|
31
|
+
return Promise.resolve(ok({ words: options.transcripts?.[name] ?? [], language: input.language }));
|
|
32
|
+
},
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface FakeIsolationProvider extends IsolationProvider {
|
|
37
|
+
readonly calls: string[];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Returns the input file's bytes unchanged: a valid audio file, so a pipeline with it still renders. */
|
|
41
|
+
export function createFakeIsolationProvider(): FakeIsolationProvider {
|
|
42
|
+
const calls: string[] = [];
|
|
43
|
+
return {
|
|
44
|
+
id: "fake",
|
|
45
|
+
calls,
|
|
46
|
+
isolate(input) {
|
|
47
|
+
calls.push(input.audio);
|
|
48
|
+
return Promise.resolve(ok(readFileSync(input.audio)));
|
|
49
|
+
},
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface FakeMusicProvider extends MusicProvider {
|
|
54
|
+
readonly calls: ({ kind: "compose"; prompt: string; seconds: number } | { kind: "video"; video: string; description: string; tags: string[] })[];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Returns `audio` (by default a few bytes that are not audio: enough for tests that never decode it). */
|
|
58
|
+
export function createFakeMusicProvider(audio: Buffer = Buffer.from("fake music")): FakeMusicProvider {
|
|
59
|
+
const calls: FakeMusicProvider["calls"] = [];
|
|
60
|
+
return {
|
|
61
|
+
id: "fake",
|
|
62
|
+
model: "fake-music",
|
|
63
|
+
calls,
|
|
64
|
+
compose(input) {
|
|
65
|
+
calls.push({ kind: "compose", ...input });
|
|
66
|
+
return Promise.resolve(ok(audio));
|
|
67
|
+
},
|
|
68
|
+
scoreVideo(input) {
|
|
69
|
+
calls.push({ kind: "video", ...input });
|
|
70
|
+
return Promise.resolve(ok(audio));
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const round = (value: number): number => Math.round(value * 1000) / 1000 || 0;
|
|
76
|
+
/** Silence kept before the first and after the last placeholder word. */
|
|
77
|
+
const PLACEHOLDER_MARGIN = 0.3;
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* The free stand-in transcript of `--placeholder`: the script's words spread evenly over the clip, each lasting 80 %
|
|
81
|
+
* of its slot. No script, no words: the clip is kept whole.
|
|
82
|
+
*/
|
|
83
|
+
export function makePlaceholderTranscript(script: string | null, duration: number): TimedWord[] {
|
|
84
|
+
const texts = (script ?? "").trim().split(/\s+/).filter((text) => text.length > 0);
|
|
85
|
+
const span = duration - 2 * PLACEHOLDER_MARGIN;
|
|
86
|
+
if (texts.length === 0 || span <= 0) return [];
|
|
87
|
+
const slot = span / texts.length;
|
|
88
|
+
return texts.map((text, index) => {
|
|
89
|
+
const start = PLACEHOLDER_MARGIN + index * slot;
|
|
90
|
+
return { text, start: round(start), end: round(start + slot * 0.8) };
|
|
91
|
+
});
|
|
92
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import type { VideoFormat } from "../compose/timeline.js";
|
|
2
|
+
import type { EndCard } from "../film.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* A footage film: filmed clips (a person talking to camera, b-roll) cut by their transcript into a captioned
|
|
6
|
+
* vertical film. Built from `marketing.json` (`footage[]`) or passed to `renderFootageFilm` as it is.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/** `talk`: someone speaks, so the cut follows the words. `broll`: pictures without speech, kept whole with a slow zoom. */
|
|
10
|
+
export const CLIP_ROLES = ["talk", "broll"] as const;
|
|
11
|
+
|
|
12
|
+
export type ClipRole = (typeof CLIP_ROLES)[number];
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* How a clip whose aspect differs from the film's fills the frame: `crop` (scaled to cover, edges cut), `blur` (the
|
|
16
|
+
* whole picture over a blurred, enlarged copy of itself) or `auto` (crop when the aspects are within 15 %, else blur).
|
|
17
|
+
*/
|
|
18
|
+
export const CLIP_FITS = ["auto", "crop", "blur"] as const;
|
|
19
|
+
|
|
20
|
+
export type ClipFit = (typeof CLIP_FITS)[number];
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* How a segment of the cut starts: `cut` (hard), `fade` (a short cross-fade from the previous segment), `zoom` (a
|
|
24
|
+
* punch-in that hides a jump cut inside one talking clip) or `whoosh` (a hard cut with the whoosh sound).
|
|
25
|
+
*/
|
|
26
|
+
export const CUT_TRANSITIONS = ["cut", "fade", "zoom", "whoosh"] as const;
|
|
27
|
+
|
|
28
|
+
export type CutTransition = (typeof CUT_TRANSITIONS)[number];
|
|
29
|
+
|
|
30
|
+
export interface FootageClip {
|
|
31
|
+
/** Unique within the film; the EDL refers to the clip by it. */
|
|
32
|
+
id: string;
|
|
33
|
+
/** A local file (absolute path) or an `https://` URL, downloaded once into the build folder. */
|
|
34
|
+
source: string;
|
|
35
|
+
role: ClipRole;
|
|
36
|
+
fit: ClipFit;
|
|
37
|
+
/** The part of the source the film uses, in seconds; `end: null` is the source's end. */
|
|
38
|
+
trim: { start: number; end: number | null };
|
|
39
|
+
/** What the clip says, for `--placeholder` only: spread evenly over the clip as a free stand-in transcript. */
|
|
40
|
+
script: string | null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Where the music under the film comes from. `prompt` and `video` are paid (ElevenLabs Music, Video to Music). */
|
|
44
|
+
export type MusicSource =
|
|
45
|
+
| { kind: "none" }
|
|
46
|
+
| { kind: "file"; path: string }
|
|
47
|
+
| { kind: "prompt"; prompt: string }
|
|
48
|
+
| { kind: "video"; description: string; tags: string[] };
|
|
49
|
+
|
|
50
|
+
export interface FootageMusic {
|
|
51
|
+
source: MusicSource;
|
|
52
|
+
/** The music's volume while nobody speaks (0-1). */
|
|
53
|
+
volume: number;
|
|
54
|
+
/** The music's volume under speech (0-1). */
|
|
55
|
+
duckedVolume: number;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** How the transcript turns into the cut (`buildEdl`). */
|
|
59
|
+
export interface CutOptions {
|
|
60
|
+
/** Words cut out wherever they stand, compared without case, punctuation and repeated letters. */
|
|
61
|
+
fillers: string[];
|
|
62
|
+
/** A silence between two words longer than this is cut (seconds). */
|
|
63
|
+
pauseSeconds: number;
|
|
64
|
+
/** Seconds kept before the first and after the last word of every kept run. */
|
|
65
|
+
padSeconds: number;
|
|
66
|
+
/** The cut's longest length in seconds; later segments are shortened or dropped. Null: no limit. */
|
|
67
|
+
maxSeconds: number | null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Default transitions by kind of cut; a segment of the EDL may name its own. */
|
|
71
|
+
export interface FootageTransitions {
|
|
72
|
+
/** A cut inside one talking clip (a removed filler or pause). */
|
|
73
|
+
jump: CutTransition;
|
|
74
|
+
/** A cut from one clip to another. */
|
|
75
|
+
clip: CutTransition;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export interface FootageFilm {
|
|
79
|
+
id: string;
|
|
80
|
+
title: string;
|
|
81
|
+
format: VideoFormat;
|
|
82
|
+
/** ISO 639 code the clips speak (`pl`, `en`): the transcript's language and the default fillers. */
|
|
83
|
+
language: string;
|
|
84
|
+
clips: FootageClip[];
|
|
85
|
+
/** Clean the talking clips' voice (ElevenLabs Voice Isolator, paid) before transcribing them. */
|
|
86
|
+
isolateVoice: boolean;
|
|
87
|
+
cut: CutOptions;
|
|
88
|
+
transitions: FootageTransitions;
|
|
89
|
+
music: FootageMusic;
|
|
90
|
+
/** Word-by-word captions from the transcript. */
|
|
91
|
+
captions: boolean;
|
|
92
|
+
/** The closing card after the last segment; null: the film ends with its last segment. */
|
|
93
|
+
endCard: EndCard | null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Filler words per language, written with repeated letters collapsed (`yyy` and `yyyy` are both `y`). */
|
|
97
|
+
export const DEFAULT_FILLERS: Record<string, string[]> = {
|
|
98
|
+
pl: ["y", "e", "yhm", "yh", "ym", "em", "hm", "mhm", "eh", "uh"],
|
|
99
|
+
en: ["um", "uh", "uhm", "er", "erm", "hm", "mhm", "ah"],
|
|
100
|
+
};
|
|
101
|
+
|
|
102
|
+
/** The default fillers of a language; none for a language without a list. */
|
|
103
|
+
export function getDefaultFillers(language: string): string[] {
|
|
104
|
+
return DEFAULT_FILLERS[language] ?? [];
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export const DEFAULT_CUT: Omit<CutOptions, "fillers"> = { pauseSeconds: 0.6, padSeconds: 0.12, maxSeconds: null };
|
|
108
|
+
|
|
109
|
+
export const DEFAULT_TRANSITIONS: FootageTransitions = { jump: "zoom", clip: "fade" };
|
|
110
|
+
|
|
111
|
+
export const DEFAULT_MUSIC_VOLUME = { volume: 0.35, duckedVolume: 0.1 } as const;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { TimedWord } from "../voice/voiceover.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Music under the voice: loud between sentences, ducked while someone speaks. The envelope is a list of keyframes
|
|
5
|
+
* computed from the film's word times; ffmpeg applies it to the music file (`volumeExpression`), so the composition
|
|
6
|
+
* plays an already ducked track.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
export interface VolumeKeyframe {
|
|
10
|
+
t: number;
|
|
11
|
+
volume: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface DuckingOptions {
|
|
15
|
+
/** The music's volume between sentences and under speech. */
|
|
16
|
+
volume: number;
|
|
17
|
+
duckedVolume: number;
|
|
18
|
+
/** Seconds the music takes to go down before the first word, and to come back after the last. */
|
|
19
|
+
attack?: number;
|
|
20
|
+
release?: number;
|
|
21
|
+
/** Two words closer than this stay one stretch of speech. */
|
|
22
|
+
gap?: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
|
|
26
|
+
|
|
27
|
+
/** Stretches of speech: the words joined when less than `gap` seconds apart. */
|
|
28
|
+
export function getSpeechSpans(words: TimedWord[], gap: number): { start: number; end: number }[] {
|
|
29
|
+
const spans: { start: number; end: number }[] = [];
|
|
30
|
+
for (const word of [...words].sort((a, b) => a.start - b.start)) {
|
|
31
|
+
const last = spans.at(-1);
|
|
32
|
+
if (last !== undefined && word.start - last.end < gap) last.end = Math.max(last.end, word.end);
|
|
33
|
+
else spans.push({ start: word.start, end: word.end });
|
|
34
|
+
}
|
|
35
|
+
return spans;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The music's volume over time as keyframes (linear between them): `volume`, down to `duckedVolume` by the first word
|
|
40
|
+
* of each stretch of speech, back up `release` seconds after its last word. Stretches too close for the music to
|
|
41
|
+
* come back up merge.
|
|
42
|
+
*/
|
|
43
|
+
export function duckingKeyframes(words: TimedWord[], options: DuckingOptions): VolumeKeyframe[] {
|
|
44
|
+
const attack = options.attack ?? 0.25;
|
|
45
|
+
const release = options.release ?? 0.5;
|
|
46
|
+
const spans = getSpeechSpans(words, Math.max(options.gap ?? 0.6, attack + release));
|
|
47
|
+
const keyframes: VolumeKeyframe[] = [];
|
|
48
|
+
const push = (t: number, volume: number) => {
|
|
49
|
+
const time = r3(Math.max(0, t));
|
|
50
|
+
const last = keyframes.at(-1);
|
|
51
|
+
if (last !== undefined && time <= last.t) {
|
|
52
|
+
last.volume = volume;
|
|
53
|
+
return;
|
|
54
|
+
}
|
|
55
|
+
keyframes.push({ t: time, volume });
|
|
56
|
+
};
|
|
57
|
+
if (spans.length === 0 || (spans[0]?.start ?? 0) - attack > 0) push(0, options.volume);
|
|
58
|
+
for (const span of spans) {
|
|
59
|
+
push(span.start - attack, options.volume);
|
|
60
|
+
push(span.start, options.duckedVolume);
|
|
61
|
+
push(span.end, options.duckedVolume);
|
|
62
|
+
push(span.end + release, options.volume);
|
|
63
|
+
}
|
|
64
|
+
return keyframes;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const n = (value: number): string => String(r3(value));
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* An expression for ffmpeg's `volume` filter (`eval=frame`, time `t`): the keyframes as a sum of linear pieces, each
|
|
71
|
+
* switched on by its own time window, so a long film needs no nesting.
|
|
72
|
+
*/
|
|
73
|
+
export function volumeExpression(keyframes: VolumeKeyframe[]): string {
|
|
74
|
+
const first = keyframes[0];
|
|
75
|
+
const last = keyframes.at(-1);
|
|
76
|
+
if (first === undefined || last === undefined) return "1";
|
|
77
|
+
const pieces = [`lt(t,${n(first.t)})*${n(first.volume)}`];
|
|
78
|
+
keyframes.slice(0, -1).forEach((from, index) => {
|
|
79
|
+
const to = keyframes[index + 1] as VolumeKeyframe;
|
|
80
|
+
const slope = (to.volume - from.volume) / (to.t - from.t);
|
|
81
|
+
const value = slope === 0 ? n(from.volume) : `(${n(from.volume)}+${String(Math.round(slope * 1e6) / 1e6)}*(t-${n(from.t)}))`;
|
|
82
|
+
pieces.push(`gte(t,${n(from.t)})*lt(t,${n(to.t)})*${value}`);
|
|
83
|
+
});
|
|
84
|
+
pieces.push(`gte(t,${n(last.t)})*${n(last.volume)}`);
|
|
85
|
+
return pieces.join("+");
|
|
86
|
+
}
|