@softure-ai/marketing-kit 0.3.1 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. package/CHANGELOG.md +57 -0
  2. package/README.md +135 -14
  3. package/dist/cli/footage.d.ts.map +1 -1
  4. package/dist/cli/footage.js +15 -6
  5. package/dist/cli/footage.js.map +1 -1
  6. package/dist/config/actions-schema.d.ts +14 -14
  7. package/dist/config/config.d.ts.map +1 -1
  8. package/dist/config/config.js +5 -0
  9. package/dist/config/config.js.map +1 -1
  10. package/dist/config/schema.d.ts +92 -69
  11. package/dist/config/schema.d.ts.map +1 -1
  12. package/dist/config/schema.js +40 -4
  13. package/dist/config/schema.js.map +1 -1
  14. package/dist/config/shot-steps.d.ts +16 -16
  15. package/dist/footage/audio.d.ts +52 -0
  16. package/dist/footage/audio.d.ts.map +1 -0
  17. package/dist/footage/audio.js +100 -0
  18. package/dist/footage/audio.js.map +1 -0
  19. package/dist/footage/captions.d.ts +57 -0
  20. package/dist/footage/captions.d.ts.map +1 -0
  21. package/dist/footage/captions.js +112 -0
  22. package/dist/footage/captions.js.map +1 -0
  23. package/dist/footage/compose.d.ts +5 -5
  24. package/dist/footage/compose.d.ts.map +1 -1
  25. package/dist/footage/compose.js +31 -22
  26. package/dist/footage/compose.js.map +1 -1
  27. package/dist/footage/edl.d.ts +54 -16
  28. package/dist/footage/edl.d.ts.map +1 -1
  29. package/dist/footage/edl.js +204 -52
  30. package/dist/footage/edl.js.map +1 -1
  31. package/dist/footage/elevenlabs.d.ts +4 -1
  32. package/dist/footage/elevenlabs.d.ts.map +1 -1
  33. package/dist/footage/elevenlabs.js +18 -6
  34. package/dist/footage/elevenlabs.js.map +1 -1
  35. package/dist/footage/fake.d.ts +2 -0
  36. package/dist/footage/fake.d.ts.map +1 -1
  37. package/dist/footage/fake.js +2 -1
  38. package/dist/footage/fake.js.map +1 -1
  39. package/dist/footage/film.d.ts +35 -1
  40. package/dist/footage/film.d.ts.map +1 -1
  41. package/dist/footage/film.js +9 -1
  42. package/dist/footage/film.js.map +1 -1
  43. package/dist/footage/frames.d.ts +38 -0
  44. package/dist/footage/frames.d.ts.map +1 -0
  45. package/dist/footage/frames.js +40 -0
  46. package/dist/footage/frames.js.map +1 -0
  47. package/dist/footage/loudness.d.ts +49 -0
  48. package/dist/footage/loudness.d.ts.map +1 -0
  49. package/dist/footage/loudness.js +114 -0
  50. package/dist/footage/loudness.js.map +1 -0
  51. package/dist/footage/normalize.d.ts +6 -2
  52. package/dist/footage/normalize.d.ts.map +1 -1
  53. package/dist/footage/normalize.js +7 -9
  54. package/dist/footage/normalize.js.map +1 -1
  55. package/dist/footage/paid.d.ts +70 -0
  56. package/dist/footage/paid.d.ts.map +1 -0
  57. package/dist/footage/paid.js +172 -0
  58. package/dist/footage/paid.js.map +1 -0
  59. package/dist/footage/pipeline.d.ts +34 -10
  60. package/dist/footage/pipeline.d.ts.map +1 -1
  61. package/dist/footage/pipeline.js +294 -85
  62. package/dist/footage/pipeline.js.map +1 -1
  63. package/dist/footage/probe.d.ts +18 -4
  64. package/dist/footage/probe.d.ts.map +1 -1
  65. package/dist/footage/probe.js +18 -13
  66. package/dist/footage/probe.js.map +1 -1
  67. package/dist/footage/process.d.ts +45 -0
  68. package/dist/footage/process.d.ts.map +1 -0
  69. package/dist/footage/process.js +109 -0
  70. package/dist/footage/process.js.map +1 -0
  71. package/dist/footage/providers.d.ts +49 -5
  72. package/dist/footage/providers.d.ts.map +1 -1
  73. package/dist/footage/providers.js +4 -0
  74. package/dist/footage/providers.js.map +1 -1
  75. package/dist/footage/publish.d.ts +62 -0
  76. package/dist/footage/publish.d.ts.map +1 -0
  77. package/dist/footage/publish.js +164 -0
  78. package/dist/footage/publish.js.map +1 -0
  79. package/dist/footage/render.d.ts +5 -4
  80. package/dist/footage/render.d.ts.map +1 -1
  81. package/dist/footage/render.js +29 -17
  82. package/dist/footage/render.js.map +1 -1
  83. package/dist/footage/sentences.d.ts +44 -0
  84. package/dist/footage/sentences.d.ts.map +1 -0
  85. package/dist/footage/sentences.js +102 -0
  86. package/dist/footage/sentences.js.map +1 -0
  87. package/dist/footage/tools.d.ts +33 -0
  88. package/dist/footage/tools.d.ts.map +1 -0
  89. package/dist/footage/tools.js +108 -0
  90. package/dist/footage/tools.js.map +1 -0
  91. package/dist/footage/words.d.ts +6 -0
  92. package/dist/footage/words.d.ts.map +1 -0
  93. package/dist/footage/words.js +14 -0
  94. package/dist/footage/words.js.map +1 -0
  95. package/dist/index.d.ts +15 -6
  96. package/dist/index.d.ts.map +1 -1
  97. package/dist/index.js +15 -6
  98. package/dist/index.js.map +1 -1
  99. package/docker/footage-worker.Dockerfile +29 -0
  100. package/package.json +2 -1
  101. package/schema/edl.schema.json +115 -0
  102. package/schema/marketing.schema.json +112 -5
  103. package/src/cli/footage.ts +13 -5
  104. package/src/config/config.ts +5 -0
  105. package/src/config/schema.ts +40 -4
  106. package/src/footage/audio.ts +123 -0
  107. package/src/footage/captions.ts +146 -0
  108. package/src/footage/compose.ts +32 -24
  109. package/src/footage/edl.ts +225 -56
  110. package/src/footage/elevenlabs.ts +21 -8
  111. package/src/footage/fake.ts +4 -1
  112. package/src/footage/film.ts +43 -2
  113. package/src/footage/frames.ts +65 -0
  114. package/src/footage/loudness.ts +143 -0
  115. package/src/footage/normalize.ts +7 -9
  116. package/src/footage/paid.ts +213 -0
  117. package/src/footage/pipeline.ts +344 -89
  118. package/src/footage/probe.ts +20 -12
  119. package/src/footage/process.ts +135 -0
  120. package/src/footage/providers.ts +56 -6
  121. package/src/footage/publish.ts +180 -0
  122. package/src/footage/render.ts +29 -19
  123. package/src/footage/sentences.ts +137 -0
  124. package/src/footage/tools.ts +119 -0
  125. package/src/footage/words.ts +15 -0
  126. package/src/index.ts +84 -0
@@ -2,6 +2,10 @@ import { z } from "zod";
2
2
 
3
3
  import type { TimedWord } from "../voice/voiceover.js";
4
4
  import { CUT_TRANSITIONS, type ClipRole, type CutOptions, type CutTransition, type FootageTransitions } from "./film.js";
5
+ import { splitFilmSentences, type Sentence } from "./sentences.js";
6
+ import { isFiller } from "./words.js";
7
+
8
+ export { isFiller, normalizeWord } from "./words.js";
5
9
 
6
10
  /**
7
11
  * The edit decision list: which parts of which clips the film keeps, in order, and how each part starts. Built from
@@ -13,8 +17,20 @@ export const SEGMENT_MOTIONS = ["none", "kenburns"] as const;
13
17
 
14
18
  export type SegmentMotion = (typeof SEGMENT_MOTIONS)[number];
15
19
 
20
+ /** A segment's sound taken from another clip or span: a J- or L-cut, or b-roll over speech. */
21
+ export interface EdlAudio {
22
+ clip: string;
23
+ /** Seconds in that trimmed clip; as long as the segment. */
24
+ start: number;
25
+ end: number;
26
+ /** The picture clip's own sound under it (0-1); 0 mutes it. */
27
+ ownVolume: number;
28
+ }
29
+
16
30
  export interface EdlSegment {
17
- /** The clip's id. */
31
+ /** Unique in the cut: `s<n>` (the sentence it starts with), `r<n>` (b-roll), `w<n>` (a talking clip without words). */
32
+ id: string;
33
+ /** The clip's id (the picture, and the sound unless `audio` says otherwise). */
18
34
  clip: string;
19
35
  /** Seconds in the trimmed clip. */
20
36
  start: number;
@@ -22,30 +38,62 @@ export interface EdlSegment {
22
38
  transition: CutTransition;
23
39
  /** `kenburns`: a slow zoom over a segment without speech. */
24
40
  motion: SegmentMotion;
41
+ audio?: EdlAudio;
25
42
  }
26
43
 
27
44
  export interface Edl {
28
- version: 1;
45
+ version: 2;
29
46
  film: string;
30
47
  segments: EdlSegment[];
31
48
  }
32
49
 
33
- export const edlSchema = z.strictObject({
50
+ const SEGMENT_ID_PATTERN = /^[A-Za-z0-9._-]+$/;
51
+
52
+ const segmentFields = {
53
+ clip: z.string().min(1).describe("The clip's id: the picture, and the sound unless audio says otherwise."),
54
+ start: z.number().min(0).describe("Seconds in the trimmed clip where the segment starts."),
55
+ end: z.number().positive().describe("Seconds in the trimmed clip where the segment ends."),
56
+ transition: z.enum(CUT_TRANSITIONS).describe("How the segment starts: cut, fade (a cross-fade from the segment before), zoom (a punch-in) or whoosh."),
57
+ motion: z.enum(SEGMENT_MOTIONS).describe("kenburns: a slow zoom over the segment; none: still."),
58
+ };
59
+
60
+ export const edlSchema = z
61
+ .strictObject({
62
+ version: z.literal(2).describe("The EDL format: 2."),
63
+ film: z.string().min(1).describe("The footage film's id."),
64
+ segments: z
65
+ .array(
66
+ z.strictObject({
67
+ id: z.string().regex(SEGMENT_ID_PATTERN, "letters, digits, dot, dash or underscore").describe("The segment's id, unique in the cut: s<n> for a sentence, r<n> for b-roll."),
68
+ ...segmentFields,
69
+ audio: z
70
+ .strictObject({
71
+ clip: z.string().min(1).describe("The clip the sound comes from."),
72
+ start: z.number().min(0).describe("Seconds in that trimmed clip where the sound starts."),
73
+ end: z.number().positive().describe("Seconds in that trimmed clip where the sound ends; end - start equals the segment's length."),
74
+ ownVolume: z.number().min(0).max(1).default(0).describe("The picture clip's own sound under it (0-1, default 0: muted)."),
75
+ })
76
+ .optional()
77
+ .describe("Sound from another clip or span: a J- or L-cut, or b-roll over speech. Without it, the segment plays its own sound."),
78
+ }),
79
+ )
80
+ .min(1, "the cut needs at least one segment")
81
+ .describe("The kept parts of the clips, in film order."),
82
+ })
83
+ .describe("The edit decision list of a footage film (edl.json).");
84
+
85
+ /** Version 1 (no ids, no audio), still read from build folders and upgraded. */
86
+ const edlV1Schema = z.strictObject({
34
87
  version: z.literal(1),
35
88
  film: z.string().min(1),
36
- segments: z
37
- .array(
38
- z.strictObject({
39
- clip: z.string().min(1),
40
- start: z.number().min(0),
41
- end: z.number().positive(),
42
- transition: z.enum(CUT_TRANSITIONS),
43
- motion: z.enum(SEGMENT_MOTIONS),
44
- }),
45
- )
46
- .min(1, "the cut needs at least one segment"),
89
+ segments: z.array(z.strictObject(segmentFields)).min(1, "the cut needs at least one segment"),
47
90
  });
48
91
 
92
+ /** The EDL's JSON Schema, for an agent or an MCP tool that edits `edl.json`. */
93
+ export function getEdlJsonSchema(): Record<string, unknown> {
94
+ return { ...z.toJSONSchema(edlSchema, { io: "input" }), title: "edl.json of a footage film (@softure-ai/marketing-kit)" };
95
+ }
96
+
49
97
  /** A clip as the cut sees it: its role, its trimmed length and its transcript (times from the trimmed start). */
50
98
  export interface EdlClip {
51
99
  id: string;
@@ -59,33 +107,24 @@ export const MIN_SEGMENT_SECONDS = 0.25;
59
107
 
60
108
  const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
61
109
 
62
- /** A word reduced for comparison: lower case, letters and digits only, repeated letters collapsed (`Yyyy,` → `y`). */
63
- export function normalizeWord(text: string): string {
64
- return text
65
- .toLowerCase()
66
- .replace(/[^\p{L}\p{N}]/gu, "")
67
- .replace(/(.)\1+/gu, "$1");
68
- }
69
-
70
- /** Whether the word is one of the fillers (compared as `normalizeWord` writes both). */
71
- export function isFiller(text: string, fillers: readonly string[]): boolean {
72
- const word = normalizeWord(text);
73
- return word.length > 0 && fillers.some((filler) => normalizeWord(filler) === word);
74
- }
75
-
76
110
  interface Span {
77
111
  start: number;
78
112
  end: number;
113
+ /** The sentence the span's first word belongs to. */
114
+ sentence: string | null;
79
115
  }
80
116
 
81
- /** The kept spans of a talking clip: runs of spoken words split at fillers and long pauses, padded, never into a neighbour word. */
82
- function getTalkSpans(clip: EdlClip, options: CutOptions): Span[] {
117
+ /**
118
+ * The kept spans of a talking clip: runs of spoken words split at fillers, long pauses and words of dropped sentences,
119
+ * padded, never into a neighbour word.
120
+ */
121
+ function getTalkSpans(clip: EdlClip, options: CutOptions, sentenceOf: Map<TimedWord, Sentence>, isKept: (word: TimedWord) => boolean): Span[] {
83
122
  const runs: TimedWord[][] = [];
84
123
  let run: TimedWord[] = [];
85
124
  const words = [...clip.words].sort((a, b) => a.start - b.start);
86
125
  words.forEach((word, index) => {
87
126
  const previous = words[index - 1];
88
- if (isFiller(word.text, options.fillers)) {
127
+ if (isFiller(word.text, options.fillers) || !isKept(word)) {
89
128
  if (run.length > 0) runs.push(run);
90
129
  run = [];
91
130
  return;
@@ -107,7 +146,7 @@ function getTalkSpans(clip: EdlClip, options: CutOptions): Span[] {
107
146
  const end = Math.min(clip.duration, last.end + options.padSeconds, after?.start ?? clip.duration);
108
147
  const previous = spans.at(-1);
109
148
  if (previous !== undefined && start <= previous.end + 0.01) previous.end = Math.max(previous.end, end);
110
- else spans.push({ start, end });
149
+ else spans.push({ start, end, sentence: sentenceOf.get(first)?.id ?? null });
111
150
  }
112
151
  return spans.filter((span) => span.end - span.start >= MIN_SEGMENT_SECONDS);
113
152
  }
@@ -127,47 +166,176 @@ function limitLength(segments: EdlSegment[], maxSeconds: number | null): EdlSegm
127
166
  return kept;
128
167
  }
129
168
 
169
+ export interface BuildEdlOptions {
170
+ /** The sentence ids (`splitFilmSentences`) to keep; the words of every other sentence are cut. Absent: all. */
171
+ keepSentences?: readonly string[];
172
+ /** Mark b-roll segments `kenburns` (a slow zoom); false keeps them still. */
173
+ isKenBurns?: boolean;
174
+ }
175
+
176
+ /** A unique id from a base: `s3`, then `s3.2`, `s3.3` for later segments of the same sentence. */
177
+ function claimId(base: string, used: Set<string>): string {
178
+ let id = base;
179
+ for (let n = 2; used.has(id); n++) id = `${base}.${n}`;
180
+ used.add(id);
181
+ return id;
182
+ }
183
+
130
184
  /**
131
185
  * The cut from the transcripts. A talking clip keeps its runs of words: fillers are cut out, a pause longer than
132
- * `pauseSeconds` is cut, `padSeconds` stays around each cut. A talking clip without words, and every b-roll clip, is
133
- * kept whole (b-roll with a slow zoom). The first segment starts with a cut; later ones with `transitions.jump`
134
- * inside one clip and `transitions.clip` between clips.
186
+ * `pauseSeconds` is cut, `padSeconds` stays around each cut, and with `keepSentences` only those sentences stay. A
187
+ * talking clip without words, and every b-roll clip, is kept whole. The first segment starts with a cut; later ones
188
+ * with `transitions.jump` inside one clip and `transitions.clip` between clips. Each segment's id names the sentence
189
+ * it starts with (`s4`), or the b-roll (`r1`) or wordless clip (`w1`) it shows.
135
190
  */
136
- export function buildEdl(filmId: string, clips: EdlClip[], options: CutOptions, transitions: FootageTransitions): Edl {
191
+ export function buildEdl(filmId: string, clips: EdlClip[], options: CutOptions, transitions: FootageTransitions, edit: BuildEdlOptions = {}): Edl {
192
+ const sentences = splitFilmSentences(clips);
193
+ const sentenceOf = new Map<TimedWord, Sentence>();
194
+ for (const sentence of sentences) for (const word of sentence.words) sentenceOf.set(word, sentence);
195
+ const keep = edit.keepSentences === undefined ? null : new Set(edit.keepSentences);
196
+ const isKept = (word: TimedWord): boolean => keep === null || keep.has(sentenceOf.get(word)?.id ?? "");
137
197
  const segments: EdlSegment[] = [];
198
+ const used = new Set<string>();
199
+ let broll = 0;
200
+ let wordless = 0;
138
201
  for (const clip of clips) {
139
- const spans = clip.role === "talk" ? getTalkSpans(clip, options) : [];
140
- const kept = spans.length > 0 ? spans : [{ start: 0, end: clip.duration }];
141
- for (const span of kept) {
202
+ const hasWords = clip.role === "talk" && clip.words.length > 0;
203
+ const spans: Span[] = hasWords ? getTalkSpans(clip, options, sentenceOf, isKept) : [{ start: 0, end: clip.duration, sentence: null }];
204
+ // A talking clip whose sentences were all dropped (or cut to nothing) leaves the film.
205
+ for (const span of spans) {
142
206
  const previous = segments.at(-1);
143
207
  const transition = previous === undefined ? "cut" : previous.clip === clip.id ? transitions.jump : transitions.clip;
144
- segments.push({ clip: clip.id, start: r3(span.start), end: r3(span.end), transition, motion: clip.role === "broll" ? "kenburns" : "none" });
208
+ const base = span.sentence ?? (clip.role === "broll" ? `r${++broll}` : `w${++wordless}`);
209
+ const motion = clip.role === "broll" && edit.isKenBurns === true ? "kenburns" : "none";
210
+ segments.push({ id: claimId(base, used), clip: clip.id, start: r3(span.start), end: r3(span.end), transition, motion });
145
211
  }
146
212
  }
147
- return { version: 1, film: filmId, segments: limitLength(segments, options.maxSeconds) };
213
+ return { version: 2, film: filmId, segments: limitLength(segments, options.maxSeconds) };
148
214
  }
149
215
 
150
- /** What makes an EDL (a hand-edited one above all) unusable with these clips; empty when it is fine. */
151
- export function findEdlProblems(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[], filmId: string): string[] {
216
+ /** The clip and span a segment's sound comes from: its `audio`, or its own picture span. */
217
+ export function getSoundSpan(segment: EdlSegment): { clip: string; start: number; end: number } {
218
+ return segment.audio ?? { clip: segment.clip, start: segment.start, end: segment.end };
219
+ }
220
+
221
+ /**
222
+ * What makes an EDL (a hand-edited one above all) unusable with these clips; empty when it is fine: another film,
223
+ * an unknown clip, a span past a clip's end, a too short segment, a repeated id, a sound span of another length.
224
+ */
225
+ export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[], filmId: string): string[] {
152
226
  const problems: string[] = [];
153
227
  if (edl.film !== filmId) problems.push(`it belongs to film "${edl.film}", not "${filmId}"`);
154
228
  const durations = new Map(clips.map((clip) => [clip.id, clip.duration]));
229
+ const seen = new Set<string>();
230
+ const checkSpan = (at: string, span: { clip: string; start: number; end: number }) => {
231
+ const duration = durations.get(span.clip);
232
+ if (duration === undefined) problems.push(`${at}.clip: no clip "${span.clip}" in the film`);
233
+ else if (span.end > duration + 0.05) problems.push(`${at}.end: ${span.end} s is past the clip's end (${duration.toFixed(2)} s)`);
234
+ };
155
235
  edl.segments.forEach((segment, index) => {
156
- const duration = durations.get(segment.clip);
157
236
  const at = `segments[${index}]`;
158
- if (duration === undefined) problems.push(`${at}.clip: no clip "${segment.clip}" in the film`);
159
- else if (segment.end > duration + 0.05) problems.push(`${at}.end: ${segment.end} s is past the clip's end (${duration.toFixed(2)} s)`);
237
+ if (seen.has(segment.id)) problems.push(`${at}.id: "${segment.id}" appears twice`);
238
+ seen.add(segment.id);
239
+ checkSpan(at, segment);
160
240
  if (segment.end - segment.start < MIN_SEGMENT_SECONDS) problems.push(`${at}: shorter than ${MIN_SEGMENT_SECONDS} s (start ${segment.start}, end ${segment.end})`);
241
+ const { audio } = segment;
242
+ if (audio === undefined) return;
243
+ checkSpan(`${at}.audio`, audio);
244
+ const length = r3(segment.end - segment.start);
245
+ const soundLength = r3(audio.end - audio.start);
246
+ if (Math.abs(length - soundLength) > 0.05) problems.push(`${at}.audio: ${soundLength} s of sound for ${length} s of picture (end - start must match)`);
161
247
  });
162
248
  return problems;
163
249
  }
164
250
 
165
- /** Reads an EDL file's JSON; an error names the first problems by path. */
251
+ /** `validateEdl` under its first name. */
252
+ export const findEdlProblems = validateEdl;
253
+
254
+ /** Reads an EDL file's JSON (version 2, or version 1 upgraded with generated ids); an error names the first problems by path. */
166
255
  export function readEdl(json: unknown): { ok: true; edl: Edl } | { ok: false; error: string } {
256
+ const isVersion1 = typeof json === "object" && json !== null && (json as { version?: unknown }).version === 1;
257
+ if (isVersion1) {
258
+ const old = edlV1Schema.safeParse(json);
259
+ if (!old.success) return { ok: false, error: formatIssues(old.error.issues) };
260
+ const used = new Set<string>();
261
+ return { ok: true, edl: { version: 2, film: old.data.film, segments: old.data.segments.map((segment, index) => ({ id: claimId(`seg${index + 1}`, used), ...segment })) } };
262
+ }
167
263
  const parsed = edlSchema.safeParse(json);
168
- if (parsed.success) return { ok: true, edl: parsed.data };
169
- const issues = parsed.error.issues.slice(0, 5).map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`);
170
- return { ok: false, error: issues.join("; ") };
264
+ return parsed.success ? { ok: true, edl: parsed.data } : { ok: false, error: formatIssues(parsed.error.issues) };
265
+ }
266
+
267
+ function formatIssues(issues: z.core.$ZodIssue[]): string {
268
+ return issues
269
+ .slice(0, 5)
270
+ .map((issue) => `${issue.path.join(".") || "(root)"}: ${issue.message}`)
271
+ .join("; ");
272
+ }
273
+
274
+ const formatSpan = (span: { start: number; end: number }): string => `${span.start}–${span.end} s`;
275
+
276
+ /** The ids of `order` that keep their relative order (a longest increasing run by their place in `before`). */
277
+ function findStayingIds(order: string[], before: Map<string, number>): Set<string> {
278
+ const places = order.map((id) => before.get(id) ?? 0);
279
+ const lengths = places.map(() => 1);
280
+ const links = places.map(() => -1);
281
+ places.forEach((place, i) => {
282
+ for (let j = 0; j < i; j++) {
283
+ if ((places[j] ?? 0) < place && (lengths[j] ?? 0) + 1 > (lengths[i] ?? 0)) {
284
+ lengths[i] = (lengths[j] ?? 0) + 1;
285
+ links[i] = j;
286
+ }
287
+ }
288
+ });
289
+ const staying = new Set<string>();
290
+ let index = lengths.indexOf(Math.max(0, ...lengths));
291
+ while (index !== -1) {
292
+ staying.add(order[index] as string);
293
+ index = links[index] ?? -1;
294
+ }
295
+ return staying;
296
+ }
297
+
298
+ /** The segment of `edl` whose sound or picture covers the middle of a sound span, to name a J/L-cut or b-roll over speech. */
299
+ function findSpeaker(edl: Edl, span: { clip: string; start: number; end: number }, self: string): string | null {
300
+ const middle = (span.start + span.end) / 2;
301
+ const match = edl.segments.find((segment) => segment.id !== self && segment.clip === span.clip && segment.start <= middle && middle < segment.end);
302
+ return match?.id ?? null;
303
+ }
304
+
305
+ /** What an edit changed, from `before` to `after`, as short lines ("dropped s3", "moved s7 to the start", "r1 over s4"). */
306
+ export function diffEdl(before: Edl, after: Edl): string[] {
307
+ const lines: string[] = [];
308
+ const old = new Map(before.segments.map((segment) => [segment.id, segment]));
309
+ const next = new Map(after.segments.map((segment) => [segment.id, segment]));
310
+ for (const segment of before.segments) if (!next.has(segment.id)) lines.push(`dropped ${segment.id}`);
311
+ for (const segment of after.segments) if (!old.has(segment.id)) lines.push(`added ${segment.id} (${segment.clip} ${formatSpan(segment)})`);
312
+ const common = after.segments.filter((segment) => old.has(segment.id)).map((segment) => segment.id);
313
+ const places = new Map(before.segments.filter((segment) => next.has(segment.id)).map((segment, index) => [segment.id, index]));
314
+ const staying = findStayingIds(common, places);
315
+ common.forEach((id, index) => {
316
+ if (staying.has(id)) return;
317
+ lines.push(index === 0 ? `moved ${id} to the start` : `moved ${id} after ${common[index - 1] ?? ""}`);
318
+ });
319
+ for (const id of common) {
320
+ const a = old.get(id) as EdlSegment;
321
+ const b = next.get(id) as EdlSegment;
322
+ if (a.clip !== b.clip || a.start !== b.start || a.end !== b.end) lines.push(`${id}: ${a.clip} ${formatSpan(a)} → ${b.clip} ${formatSpan(b)}`);
323
+ if (a.transition !== b.transition) lines.push(`${id}: ${a.transition} → ${b.transition}`);
324
+ if (a.motion !== b.motion) lines.push(`${id}: motion ${a.motion} → ${b.motion}`);
325
+ }
326
+ for (const segment of after.segments) {
327
+ const was = old.get(segment.id)?.audio;
328
+ const { audio } = segment;
329
+ if (audio === undefined) {
330
+ if (was !== undefined) lines.push(`${segment.id} plays its own sound again`);
331
+ continue;
332
+ }
333
+ if (was !== undefined && was.clip === audio.clip && was.start === audio.start && was.end === audio.end && was.ownVolume === audio.ownVolume) continue;
334
+ const speaker = findSpeaker(after, audio, segment.id) ?? findSpeaker(before, audio, segment.id);
335
+ const source = speaker ?? `${audio.clip} ${formatSpan(audio)}`;
336
+ lines.push(`${segment.id} over ${source}${audio.ownVolume > 0 ? ` (own sound at ${Math.round(audio.ownVolume * 100)} %)` : ""}`);
337
+ }
338
+ return lines;
171
339
  }
172
340
 
173
341
  export function getEdlDuration(edl: Edl): number {
@@ -186,18 +354,19 @@ export function getSegmentStarts(edl: Edl): number[] {
186
354
  }
187
355
 
188
356
  /**
189
- * The words the film says, in film time: each clip's words that fall in a kept segment (by their middle), shifted by
190
- * where the segment lands; fillers left out. Every cut shifts every later word.
357
+ * The words the film says, in film time: the words of each segment's sound span (its own, or `audio`) that fall in it
358
+ * (by their middle), shifted by where the segment lands; fillers left out. Every cut shifts every later word.
191
359
  */
192
360
  export function remapWords(edl: Edl, words: Record<string, TimedWord[]>, fillers: readonly string[]): TimedWord[] {
193
361
  const starts = getSegmentStarts(edl);
194
362
  const film: TimedWord[] = [];
195
363
  edl.segments.forEach((segment, index) => {
196
- const offset = (starts[index] as number) - segment.start;
197
- for (const word of words[segment.clip] ?? []) {
364
+ const sound = getSoundSpan(segment);
365
+ const offset = (starts[index] as number) - sound.start;
366
+ for (const word of words[sound.clip] ?? []) {
198
367
  const middle = (word.start + word.end) / 2;
199
- if (middle < segment.start || middle >= segment.end || isFiller(word.text, fillers)) continue;
200
- film.push({ text: word.text, start: r3(Math.max(word.start, segment.start) + offset), end: r3(Math.min(word.end, segment.end) + offset) });
368
+ if (middle < sound.start || middle >= sound.end || isFiller(word.text, fillers)) continue;
369
+ film.push({ text: word.text, start: r3(Math.max(word.start, sound.start) + offset), end: r3(Math.min(word.end, sound.end) + offset) });
201
370
  }
202
371
  });
203
372
  return film;
@@ -6,12 +6,13 @@ import { z } from "zod";
6
6
  import { ELEVENLABS_API_KEY_ENV } from "../voice/elevenlabs.js";
7
7
  import { err, ok, type TtsResult } from "../voice/provider.js";
8
8
  import { ELEVENLABS_API_URL, type TimedWord } from "../voice/voiceover.js";
9
- import type { IsolationProvider, MusicProvider, SttProvider, SttTranscript } from "./providers.js";
9
+ import type { IsolationProvider, MusicProvider, PaidAudio, SttProvider, SttTranscript } from "./providers.js";
10
10
 
11
11
  /**
12
12
  * ElevenLabs behind the footage providers (API reference checked 2026-10-29): Scribe speech to text
13
13
  * (`/v1/speech-to-text`, word timestamps), Voice Isolator (`/v1/audio-isolation`), Music (`/v1/music`) and Video to
14
- * Music (`/v1/music/video-to-music`). Each takes an injected `fetch`, so tests reach no network.
14
+ * Music (`/v1/music/video-to-music`). Each takes an injected `fetch`, so tests reach no network, and passes the
15
+ * caller's `signal` to it. A `character-cost` header, when a response carries one, becomes the call's `credits`.
15
16
  */
16
17
 
17
18
  export const ELEVENLABS_STT_MODEL = "scribe_v2";
@@ -71,10 +72,20 @@ async function send(options: ElevenLabsFootageOptions, label: string, path: stri
71
72
  return ok(response);
72
73
  }
73
74
 
74
- async function readAudio(label: string, response: Response): Promise<TtsResult<Buffer>> {
75
+ /** The charge a response reports in its `character-cost` header; undefined when it reports none. */
76
+ export function readReportedCredits(response: Response): number | undefined {
77
+ const header = response.headers.get("character-cost");
78
+ if (header === null || header.trim().length === 0) return undefined;
79
+ const credits = Number(header);
80
+ return Number.isFinite(credits) ? credits : undefined;
81
+ }
82
+
83
+ const withCredits = <T extends object>(value: T, credits: number | undefined): T & { credits?: number } => (credits === undefined ? value : { ...value, credits });
84
+
85
+ async function readAudio(label: string, response: Response): Promise<TtsResult<PaidAudio>> {
75
86
  try {
76
87
  const audio = Buffer.from(await response.arrayBuffer());
77
- return audio.length > 0 ? ok(audio) : err(`ElevenLabs ${label}: the response has no audio.`);
88
+ return audio.length > 0 ? ok(withCredits({ audio }, readReportedCredits(response))) : err(`ElevenLabs ${label}: the response has no audio.`);
78
89
  } catch (error) {
79
90
  return err(`ElevenLabs ${label}: reading the audio failed (${String(error)}).`);
80
91
  }
@@ -92,7 +103,7 @@ export function createElevenLabsSttProvider(options: ElevenLabsFootageOptions):
92
103
  form.append("language_code", input.language);
93
104
  form.append("timestamps_granularity", "word");
94
105
  form.append("tag_audio_events", "false");
95
- const response = await send(options, "speech to text", "/v1/speech-to-text", { method: "POST", body: form });
106
+ const response = await send(options, "speech to text", "/v1/speech-to-text", { method: "POST", body: form, signal: input.signal ?? null });
96
107
  if (!response.ok) return response;
97
108
  let body: unknown;
98
109
  try {
@@ -100,7 +111,8 @@ export function createElevenLabsSttProvider(options: ElevenLabsFootageOptions):
100
111
  } catch (error) {
101
112
  return err(`ElevenLabs speech to text: the response is not JSON (${String(error)}).`);
102
113
  }
103
- return readScribeResponse(body, input.language);
114
+ const transcript = readScribeResponse(body, input.language);
115
+ return transcript.ok ? ok(withCredits(transcript.value, readReportedCredits(response.value))) : transcript;
104
116
  },
105
117
  };
106
118
  }
@@ -111,7 +123,7 @@ export function createElevenLabsIsolationProvider(options: ElevenLabsFootageOpti
111
123
  async isolate(input) {
112
124
  const form = new FormData();
113
125
  form.append("audio", fileBlob(input.audio), basename(input.audio));
114
- const response = await send(options, "voice isolation", "/v1/audio-isolation", { method: "POST", body: form });
126
+ const response = await send(options, "voice isolation", "/v1/audio-isolation", { method: "POST", body: form, signal: input.signal ?? null });
115
127
  return response.ok ? readAudio("voice isolation", response.value) : response;
116
128
  },
117
129
  };
@@ -133,6 +145,7 @@ export function createElevenLabsMusicProvider(options: ElevenLabsFootageOptions)
133
145
  method: "POST",
134
146
  headers: { "Content-Type": "application/json" },
135
147
  body: JSON.stringify(body),
148
+ signal: input.signal ?? null,
136
149
  });
137
150
  return response.ok ? readAudio("music", response.value) : response;
138
151
  },
@@ -141,7 +154,7 @@ export function createElevenLabsMusicProvider(options: ElevenLabsFootageOptions)
141
154
  form.append("videos", fileBlob(input.video), basename(input.video));
142
155
  if (input.description.length > 0) form.append("description", input.description);
143
156
  for (const tag of input.tags) form.append("tags", tag);
144
- const response = await send(options, "video to music", "/v1/music/video-to-music?output_format=mp3_44100_128", { method: "POST", body: form });
157
+ const response = await send(options, "video to music", "/v1/music/video-to-music?output_format=mp3_44100_128", { method: "POST", body: form, signal: input.signal ?? null });
145
158
  return response.ok ? readAudio("video to music", response.value) : response;
146
159
  },
147
160
  };
@@ -12,6 +12,8 @@ export interface FakeSttOptions {
12
12
  transcripts?: Record<string, TimedWord[]>;
13
13
  /** Makes every call fail with this message. */
14
14
  failWith?: string;
15
+ /** The charge every call reports, as a provider whose API returns one. */
16
+ credits?: number;
15
17
  }
16
18
 
17
19
  export interface FakeSttProvider extends SttProvider {
@@ -28,7 +30,8 @@ export function createFakeSttProvider(options: FakeSttOptions = {}): FakeSttProv
28
30
  calls.push(input);
29
31
  if (options.failWith !== undefined) return Promise.resolve({ ok: false, error: options.failWith });
30
32
  const name = basename(input.audio).split(".")[0] ?? "";
31
- return Promise.resolve(ok({ words: options.transcripts?.[name] ?? [], language: input.language }));
33
+ const transcript = { words: options.transcripts?.[name] ?? [], language: input.language };
34
+ return Promise.resolve(ok(options.credits === undefined ? transcript : { ...transcript, credits: options.credits }));
32
35
  },
33
36
  };
34
37
  }
@@ -1,5 +1,6 @@
1
1
  import type { VideoFormat } from "../compose/timeline.js";
2
2
  import type { EndCard } from "../film.js";
3
+ import type { LoudnessTarget } from "./loudness.js";
3
4
 
4
5
  /**
5
6
  * A footage film: filmed clips (a person talking to camera, b-roll) cut by their transcript into a captioned
@@ -87,10 +88,23 @@ export interface FootageFilm {
87
88
  cut: CutOptions;
88
89
  transitions: FootageTransitions;
89
90
  music: FootageMusic;
90
- /** Word-by-word captions from the transcript. */
91
+ /** Captions from the transcript. */
91
92
  captions: boolean;
93
+ /** `phrase` (the default): 2-6 words of plain text with a soft shadow; `highlight`: a pill that lights each word as it is said. */
94
+ captionStyle?: CaptionStyle;
95
+ /** Places the 9:16 caption box inside a platform's safe zone (`CAPTION_SAFE_ZONES`); absent: the layout as it is. */
96
+ captionPreset?: CaptionPreset;
97
+ /** The cut's look: zoom size, fade length, Ken Burns, the end card's whoosh. Absent fields take `DEFAULT_EDIT`. */
98
+ edit?: Partial<FootageEdit>;
92
99
  /** The closing card after the last segment; null: the film ends with its last segment. */
93
100
  endCard: EndCard | null;
101
+ /** The speech track's loudness, normalized before the music is mixed under it. Absent: −14 LUFS, −1 dBTP; null: left as filmed. */
102
+ loudness?: LoudnessTarget | null;
103
+ /**
104
+ * The cover JPG written next to the film. Absent or without `at`: the middle of the first talking segment; `at`:
105
+ * that second of the film; null: no cover.
106
+ */
107
+ cover?: { at?: number } | null;
94
108
  }
95
109
 
96
110
  /** Filler words per language, written with repeated letters collapsed (`yyy` and `yyyy` are both `y`). */
@@ -106,6 +120,33 @@ export function getDefaultFillers(language: string): string[] {
106
120
 
107
121
  export const DEFAULT_CUT: Omit<CutOptions, "fillers"> = { pauseSeconds: 0.6, padSeconds: 0.12, maxSeconds: null };
108
122
 
109
- export const DEFAULT_TRANSITIONS: FootageTransitions = { jump: "zoom", clip: "fade" };
123
+ /** Hard cuts everywhere: an ordinary phone video, not a promo. */
124
+ export const DEFAULT_TRANSITIONS: FootageTransitions = { jump: "cut", clip: "cut" };
125
+
126
+ export const CAPTION_STYLES = ["phrase", "highlight"] as const;
127
+
128
+ export type CaptionStyle = (typeof CAPTION_STYLES)[number];
129
+
130
+ export const CAPTION_PRESETS = ["reels", "shorts", "tiktok", "all"] as const;
131
+
132
+ export type CaptionPreset = (typeof CAPTION_PRESETS)[number];
133
+
134
+ export interface FootageEdit {
135
+ /** How far a `zoom` cut punches in, in percent of the frame (105-110). */
136
+ zoomPercent: number;
137
+ /** A `fade` cut's length in frames at 30 fps (1-30). */
138
+ fadeFrames: number;
139
+ /** How far b-roll zooms over its length (0-5 %); 0 keeps b-roll still and `buildEdl` marks none `kenburns`. */
140
+ kenBurnsPercent: number;
141
+ /** Play the whoosh (and pop) as the end card comes in. */
142
+ isEndCardWhoosh: boolean;
143
+ }
144
+
145
+ export const DEFAULT_EDIT: FootageEdit = { zoomPercent: 108, fadeFrames: 10, kenBurnsPercent: 0, isEndCardWhoosh: false };
146
+
147
+ /** The film's edit settings with the defaults filled in. */
148
+ export function getFootageEdit(film: Pick<FootageFilm, "edit">): FootageEdit {
149
+ return { ...DEFAULT_EDIT, ...film.edit };
150
+ }
110
151
 
111
152
  export const DEFAULT_MUSIC_VOLUME = { volume: 0.35, duckedVolume: 0.1 } as const;
@@ -0,0 +1,65 @@
1
+ import { mkdirSync } from "node:fs";
2
+ import { join } from "node:path";
3
+
4
+ import { probeFile } from "./probe.js";
5
+ import { runFfmpeg } from "./process.js";
6
+
7
+ /**
8
+ * Frames of a clip as JPG files (a contact sheet), so an agent can look at the footage and choose the opening shot or
9
+ * the b-roll. Read from the normalized clip, so the frames are upright and in SDR as the film will show them.
10
+ */
11
+
12
+ export type FrameSpacing = { count: number } | { everySeconds: number };
13
+
14
+ export interface ExtractFramesOptions {
15
+ /** The folder the JPGs go to (`<name>-<n>.jpg`); created when missing. */
16
+ outDir: string;
17
+ /** The frames' width in px (the height keeps the aspect); 360 by default. */
18
+ width?: number;
19
+ /** The files' name before the number; `frame` by default. */
20
+ name?: string;
21
+ signal?: AbortSignal;
22
+ }
23
+
24
+ export interface ExtractedFrame {
25
+ file: string;
26
+ /** Seconds in the clip. */
27
+ seconds: number;
28
+ }
29
+
30
+ const DEFAULT_WIDTH = 360;
31
+ /** The most frames one call writes, so a long clip with a short spacing stays a sheet, not a film. */
32
+ export const MAX_FRAMES = 120;
33
+
34
+ const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
35
+
36
+ /** The times to take frames at: `count` frames at the middles of equal parts, or one every `everySeconds` from its half. */
37
+ export function getFrameTimes(duration: number, spacing: FrameSpacing): number[] {
38
+ if (duration <= 0) return [];
39
+ const count = "count" in spacing ? spacing.count : Math.floor(duration / spacing.everySeconds);
40
+ const step = "count" in spacing ? duration / spacing.count : spacing.everySeconds;
41
+ const total = Math.min(MAX_FRAMES, Math.max(1, Math.floor(count)));
42
+ return Array.from({ length: total }, (_, index) => r3(Math.min(duration - 0.01, (index + 0.5) * step)));
43
+ }
44
+
45
+ /** One frame at `seconds`, scaled to `width`. */
46
+ export function buildFrameArgs(file: string, seconds: number, width: number, output: string): string[] {
47
+ return ["-v", "error", "-y", "-ss", String(seconds), "-i", file, "-frames:v", "1", "-vf", `scale=${width}:-2`, "-q:v", "3", output];
48
+ }
49
+
50
+ /** Writes the frames and answers their files and times; an error names the file. Rejects on abort. */
51
+ export async function extractFrames(file: string, spacing: FrameSpacing, options: ExtractFramesOptions): Promise<{ ok: true; frames: ExtractedFrame[] } | { ok: false; error: string }> {
52
+ if ("count" in spacing ? !(spacing.count >= 1) : !(spacing.everySeconds > 0)) return { ok: false, error: "frames: count must be at least 1, everySeconds above 0" };
53
+ const signal = options.signal === undefined ? {} : { signal: options.signal };
54
+ const probed = await probeFile(file, signal);
55
+ if (!probed.ok) return probed;
56
+ mkdirSync(options.outDir, { recursive: true });
57
+ const width = options.width ?? DEFAULT_WIDTH;
58
+ const frames: ExtractedFrame[] = [];
59
+ for (const [index, seconds] of getFrameTimes(probed.probe.duration, spacing).entries()) {
60
+ const output = join(options.outDir, `${options.name ?? "frame"}-${String(index + 1).padStart(3, "0")}.jpg`);
61
+ await runFfmpeg(`A frame of ${file} at ${seconds} s`, buildFrameArgs(file, seconds, width, output), { ...signal, output });
62
+ frames.push({ file: output, seconds });
63
+ }
64
+ return { ok: true, frames };
65
+ }