@softure-ai/marketing-kit 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/CHANGELOG.md +71 -0
  2. package/README.md +66 -9
  3. package/dist/cli/footage.js +1 -1
  4. package/dist/cli/footage.js.map +1 -1
  5. package/dist/compose/timeline.d.ts +23 -3
  6. package/dist/compose/timeline.d.ts.map +1 -1
  7. package/dist/compose/timeline.js +31 -11
  8. package/dist/compose/timeline.js.map +1 -1
  9. package/dist/config/actions-schema.d.ts +14 -14
  10. package/dist/config/config.d.ts.map +1 -1
  11. package/dist/config/config.js +2 -0
  12. package/dist/config/config.js.map +1 -1
  13. package/dist/config/schema.d.ts +83 -70
  14. package/dist/config/schema.d.ts.map +1 -1
  15. package/dist/config/schema.js +21 -1
  16. package/dist/config/schema.js.map +1 -1
  17. package/dist/config/shot-steps.d.ts +16 -16
  18. package/dist/footage/captions.d.ts +45 -3
  19. package/dist/footage/captions.d.ts.map +1 -1
  20. package/dist/footage/captions.js +65 -5
  21. package/dist/footage/captions.js.map +1 -1
  22. package/dist/footage/compose.d.ts.map +1 -1
  23. package/dist/footage/compose.js +8 -4
  24. package/dist/footage/compose.js.map +1 -1
  25. package/dist/footage/declick.d.ts +106 -0
  26. package/dist/footage/declick.d.ts.map +1 -0
  27. package/dist/footage/declick.js +296 -0
  28. package/dist/footage/declick.js.map +1 -0
  29. package/dist/footage/draft.d.ts +8 -3
  30. package/dist/footage/draft.d.ts.map +1 -1
  31. package/dist/footage/draft.js +55 -22
  32. package/dist/footage/draft.js.map +1 -1
  33. package/dist/footage/edl.d.ts +28 -2
  34. package/dist/footage/edl.d.ts.map +1 -1
  35. package/dist/footage/edl.js +53 -2
  36. package/dist/footage/edl.js.map +1 -1
  37. package/dist/footage/film.d.ts +29 -3
  38. package/dist/footage/film.d.ts.map +1 -1
  39. package/dist/footage/film.js +12 -0
  40. package/dist/footage/film.js.map +1 -1
  41. package/dist/footage/isolate-ranges.d.ts +37 -0
  42. package/dist/footage/isolate-ranges.d.ts.map +1 -0
  43. package/dist/footage/isolate-ranges.js +113 -0
  44. package/dist/footage/isolate-ranges.js.map +1 -0
  45. package/dist/footage/loudness.d.ts +58 -0
  46. package/dist/footage/loudness.d.ts.map +1 -1
  47. package/dist/footage/loudness.js +80 -0
  48. package/dist/footage/loudness.js.map +1 -1
  49. package/dist/footage/paid.d.ts +18 -0
  50. package/dist/footage/paid.d.ts.map +1 -1
  51. package/dist/footage/paid.js +32 -2
  52. package/dist/footage/paid.js.map +1 -1
  53. package/dist/footage/pipeline.d.ts +43 -5
  54. package/dist/footage/pipeline.d.ts.map +1 -1
  55. package/dist/footage/pipeline.js +103 -20
  56. package/dist/footage/pipeline.js.map +1 -1
  57. package/dist/footage/providers.d.ts +5 -0
  58. package/dist/footage/providers.d.ts.map +1 -1
  59. package/dist/footage/publish.d.ts +11 -2
  60. package/dist/footage/publish.d.ts.map +1 -1
  61. package/dist/footage/publish.js +14 -10
  62. package/dist/footage/publish.js.map +1 -1
  63. package/dist/footage/speech-ranges.d.ts +65 -0
  64. package/dist/footage/speech-ranges.d.ts.map +1 -0
  65. package/dist/footage/speech-ranges.js +98 -0
  66. package/dist/footage/speech-ranges.js.map +1 -0
  67. package/dist/index.d.ts +10 -7
  68. package/dist/index.d.ts.map +1 -1
  69. package/dist/index.js +10 -7
  70. package/dist/index.js.map +1 -1
  71. package/package.json +1 -1
  72. package/schema/edl.schema.json +17 -0
  73. package/schema/marketing.schema.json +43 -1
  74. package/src/cli/footage.ts +1 -1
  75. package/src/compose/timeline.ts +41 -10
  76. package/src/config/config.ts +2 -0
  77. package/src/config/schema.ts +23 -1
  78. package/src/footage/captions.ts +81 -6
  79. package/src/footage/compose.ts +8 -4
  80. package/src/footage/declick.ts +353 -0
  81. package/src/footage/draft.ts +58 -23
  82. package/src/footage/edl.ts +69 -2
  83. package/src/footage/film.ts +37 -3
  84. package/src/footage/isolate-ranges.ts +161 -0
  85. package/src/footage/loudness.ts +112 -0
  86. package/src/footage/paid.ts +40 -2
  87. package/src/footage/pipeline.ts +155 -22
  88. package/src/footage/providers.ts +2 -0
  89. package/src/footage/publish.ts +20 -11
  90. package/src/footage/speech-ranges.ts +131 -0
  91. package/src/index.ts +70 -0
@@ -2,15 +2,16 @@ import { copyFileSync, existsSync, mkdirSync, rmSync, statSync, writeFileSync }
2
2
  import { basename, dirname, join, resolve } from "node:path";
3
3
  import { fileURLToPath } from "node:url";
4
4
 
5
- import { captionChunks, type CaptionLayout, type LayoutOverride } from "../compose/timeline.js";
5
+ import type { CaptionLayout, LayoutOverride } from "../compose/timeline.js";
6
6
  import type { BrandColors } from "../config/colors.js";
7
7
  import { err, ok } from "../voice/provider.js";
8
8
  import type { TimedWord } from "../voice/voiceover.js";
9
- import { getCaptionSpans, getFootageCaptionLayout, phraseChunks } from "./captions.js";
9
+ import { getCaptionChunkLayouts, getCaptionSpans, getFootageCaptionChunks, getFootageCaptionLayout, isSameCaptionPlace, type CaptionWordsFilm } from "./captions.js";
10
10
  import { getSegmentScales, getSegmentZoomEases } from "./compose.js";
11
11
  import { chooseFootageDraftFont } from "./draft-font.js";
12
12
  import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
13
- import { getFootageEdit, type FootageFilm } from "./film.js";
13
+ import { declickSpeech, describeDeclick, type DeclickReport } from "./declick.js";
14
+ import { getDeclickLevel, getFootageEdit, type FootageFilm } from "./film.js";
14
15
  import { duckingKeyframes, getDuckingOptions } from "./music.js";
15
16
  import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
16
17
  import { findFootageMusic, getFfmpegReporting, getFrame, type FootageBrand, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
@@ -80,16 +81,25 @@ export function toAssColor(hex: string): string {
80
81
  return `&H${alpha}${pair(4)}${pair(2)}${pair(0)}`;
81
82
  }
82
83
 
83
- /** Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. */
84
+ /**
85
+ * Caption text as ASS dialogue: override braces and backslashes would be read as tags, line breaks end the line. A
86
+ * no-break space (U+00A0) becomes ASS's hard space `\h`, which libass never breaks a line at, as the render's HTML.
87
+ */
84
88
  function toAssText(text: string): string {
85
- return text.replace(/\\/g, "/").replace(/\{/g, "(").replace(/\}/g, ")").replace(/\s+/g, " ").trim();
89
+ return text
90
+ .replace(/\\/g, "/")
91
+ .replace(/\{/g, "(")
92
+ .replace(/\}/g, ")")
93
+ .replace(/[^\S\u00a0]+/g, " ")
94
+ .replace(/^ +| +$/g, "")
95
+ .replace(/\u00a0/g, "\\h");
86
96
  }
87
97
 
88
98
  /** The caption colours the draft paints with (`BrandColors`). */
89
99
  export type FootageDraftCaptionColors = Pick<BrandColors, "captionBackground" | "captionText" | "captionHighlight">;
90
100
 
91
101
  export interface FootageDraftCaptionsInput {
92
- film: Pick<FootageFilm, "captions" | "captionStyle">;
102
+ film: Pick<FootageFilm, "captions" | "captionStyle"> & CaptionWordsFilm;
93
103
  edl: Edl;
94
104
  /** The film's words in film time (`remapWords`). */
95
105
  words: TimedWord[];
@@ -107,7 +117,7 @@ export interface FootageDraftCaptionsInput {
107
117
  * A `highlight` chunk as ASS events: its span cut wherever a word lights up or goes out, each piece showing the chunk
108
118
  * with the words lit then in `highlight` (a word stays lit for the render's 0.32 s from its start).
109
119
  */
110
- function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: number; end: number }, colors: { text: string; highlight: string }): string[] {
120
+ function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: number; end: number }, colors: { text: string; highlight: string }, styleName: string): string[] {
111
121
  const cs = (seconds: number): number => Math.round(seconds * 100);
112
122
  const [from, to] = [cs(span.start), cs(span.end)];
113
123
  const lights = chunk.words.map((word) => ({ on: cs(word.start), off: cs(word.start + HIGHLIGHT_SECONDS) }));
@@ -122,7 +132,7 @@ function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: numb
122
132
  return light.on <= at && at < light.off ? `{\\1c${colors.highlight}&}${shown}{\\1c${colors.text}&}` : shown;
123
133
  })
124
134
  .join(" ");
125
- events.push(`Dialogue: 0,${toAssTime(at / 100)},${toAssTime(next / 100)},Caption,,0,0,0,,${text}`);
135
+ events.push(`Dialogue: 0,${toAssTime(at / 100)},${toAssTime(next / 100)},${styleName},,0,0,0,,${text}`);
126
136
  });
127
137
  return events;
128
138
  }
@@ -132,12 +142,13 @@ function buildHighlightEvents(chunk: { words: TimedWord[] }, span: { start: numb
132
142
  * caption box from its top (or from its foot, with `bottom`). `phrase` (the default): the film's phrases
133
143
  * (`phraseChunks`), white bold text with a dark outline and shadow. `highlight`: the render's chunks of up to four
134
144
  * words (`captionChunks`) on an opaque box of `captionBackground`, in `captionText`, the word being said in
135
- * `captionHighlight`. Null when the film has no captions or nothing is said.
145
+ * `captionHighlight`. Both keep the film's kept words with the next, as the render does. Null when the film has no
146
+ * captions or nothing is said.
136
147
  */
137
148
  export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): string | null {
138
149
  if (!input.film.captions) return null;
139
150
  const isHighlight = input.film.captionStyle === "highlight";
140
- const chunks = isHighlight ? captionChunks(input.words) : phraseChunks(input.words);
151
+ const chunks = getFootageCaptionChunks(input.film, input.words);
141
152
  if (chunks.length === 0) return null;
142
153
  const spans = getCaptionSpans(chunks, getEdlDuration(input.edl));
143
154
  const { caption, frame } = input;
@@ -156,8 +167,8 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
156
167
  outline: Math.max(1, Math.round(caption.fontSize / 16)),
157
168
  shadow: Math.max(1, Math.round(caption.fontSize / 25)),
158
169
  };
159
- const style = [
160
- "Caption",
170
+ const toStyle = (name: string, place: CaptionLayout): string => [
171
+ name,
161
172
  family,
162
173
  caption.fontSize,
163
174
  look.primary,
@@ -176,18 +187,31 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
176
187
  look.outline,
177
188
  look.shadow,
178
189
  // ASS alignment 2 hangs the text from MarginV above the bottom, 8 from MarginV below the top.
179
- caption.bottom === undefined ? 8 : 2,
180
- Math.round(caption.left),
181
- Math.round(caption.right),
182
- Math.round(caption.bottom ?? caption.top),
190
+ place.bottom === undefined ? 8 : 2,
191
+ Math.round(place.left),
192
+ Math.round(place.right),
193
+ Math.round(place.bottom ?? place.top),
183
194
  1,
184
195
  ].join(",");
196
+ // A chunk in a segment's own box gets a style of that box (Caption2, Caption3, ...); the film's box stays `Caption`.
197
+ const styles = [toStyle("Caption", caption)];
198
+ const moved: CaptionLayout[] = [];
199
+ const styleNames = getCaptionChunkLayouts(chunks, input.edl, caption).map((place) => {
200
+ if (isSameCaptionPlace(place, caption)) return "Caption";
201
+ let index = moved.findIndex((other) => isSameCaptionPlace(other, place));
202
+ if (index === -1) {
203
+ index = moved.push(place) - 1;
204
+ styles.push(toStyle(`Caption${index + 2}`, place));
205
+ }
206
+ return `Caption${index + 2}`;
207
+ });
185
208
  const highlight = { text: text.replace(/^&H../, "&H"), highlight: toAssColor(colors.captionHighlight).replace(/^&H../, "&H") };
186
209
  const events = chunks.flatMap((chunk, index) => {
187
210
  const span = spans[index] as { start: number; end: number };
188
- if (isHighlight) return buildHighlightEvents(chunk, span, highlight);
211
+ const styleName = styleNames[index] ?? "Caption";
212
+ if (isHighlight) return buildHighlightEvents(chunk, span, highlight, styleName);
189
213
  const line = toAssText(chunk.words.map((word) => word.text).join(" "));
190
- return [`Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},Caption,,0,0,0,,${line}`];
214
+ return [`Dialogue: 0,${toAssTime(span.start)},${toAssTime(span.end)},${styleName},,0,0,0,,${line}`];
191
215
  });
192
216
  return [
193
217
  "[Script Info]",
@@ -199,7 +223,7 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
199
223
  "",
200
224
  "[V4+ Styles]",
201
225
  "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding",
202
- `Style: ${style}`,
226
+ ...styles.map((style) => `Style: ${style}`),
203
227
  "",
204
228
  "[Events]",
205
229
  "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text",
@@ -427,6 +451,8 @@ export interface FootageDraft {
427
451
  width: number;
428
452
  height: number;
429
453
  bytes: number;
454
+ /** What the speech track's declick stage did, as the render does it; null when the film's declick is off. */
455
+ declick: DeclickReport | null;
430
456
  }
431
457
 
432
458
  /** The draft's music track: a file to duck, or why there is none. */
@@ -478,13 +504,16 @@ async function runRenderFootageDraft(
478
504
  const draftDir = resolve(options.buildDir, "draft");
479
505
  rmSync(draftDir, { recursive: true, force: true });
480
506
  mkdirSync(draftDir, { recursive: true });
481
- const items = music.value.kind === "file" ? ["speech", "music", "film"] : ["speech", "film"];
507
+ const declickLevel = getDeclickLevel(film);
508
+ const items = ["speech", ...(declickLevel === "off" ? [] : ["declick"]), ...(music.value.kind === "file" ? ["music"] : []), "film"];
482
509
  const report = (item: string, done: number): void => {
483
510
  const itemIndex = items.indexOf(item) + 1;
484
511
  options.onProgress?.({ step: "draft", item, itemIndex, itemCount: items.length, done: Math.min(seconds * items.length, (itemIndex - 1) * seconds + done), total: seconds * items.length });
485
512
  };
486
513
  const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, resolve(clip.file)]));
487
- const speech = join(draftDir, "speech.mp3");
514
+ const cutSpeech = join(draftDir, "speech.mp3");
515
+ // The declicked track stays WAV: the draft's one encode is its only lossy step after the cut.
516
+ const speech = declickLevel === "off" ? cutSpeech : join(draftDir, "speech.declicked.wav");
488
517
  const words = remapWords(edl, transcript.words, film.cut.fillers);
489
518
  const frame = getFrame(film);
490
519
  const brandFont = chooseFootageDraftFont(options.brand?.fonts.body ?? null);
@@ -509,7 +538,13 @@ async function runRenderFootageDraft(
509
538
  const output = resolve(options.output);
510
539
  mkdirSync(dirname(output), { recursive: true });
511
540
  report("speech", 0);
512
- await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: speech, onSeconds: (done) => report("speech", done) });
541
+ await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, cutSpeech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: cutSpeech, onSeconds: (done) => report("speech", done) });
542
+ let declick: DeclickReport | null = null;
543
+ if (declickLevel !== "off") {
544
+ report("declick", 0);
545
+ declick = await declickSpeech({ input: cutSpeech, output: speech, words, level: declickLevel, seconds, ...aborting, onSeconds: (done) => report("declick", done) });
546
+ log(describeDeclick(declick));
547
+ }
513
548
  let musicFile: string | null = null;
514
549
  if (music.value.kind === "file") {
515
550
  musicFile = join(draftDir, "music.mp3");
@@ -542,5 +577,5 @@ async function runRenderFootageDraft(
542
577
  const draftMusic: FootageDraftMusic = music.value.kind === "file" ? "played" : music.value.kind;
543
578
  const notes = [captions === null ? "no captions" : `captions in ${font.family}`, draftMusic === "played" ? "music" : draftMusic === "none" ? "no music" : "music not cached"];
544
579
  log(`draft: ${output}, ${draft.width}×${draft.height}, ${seconds.toFixed(1)} s, ${bytes} bytes, ${notes.join(", ")}.`);
545
- return ok({ output, music: draftMusic, captionFont: captions === null ? null : font.family, seconds, width: draft.width, height: draft.height, bytes });
580
+ return ok({ output, music: draftMusic, captionFont: captions === null ? null : font.family, seconds, width: draft.width, height: draft.height, bytes, declick });
546
581
  }
@@ -27,6 +27,16 @@ export interface EdlAudio {
27
27
  ownVolume: number;
28
28
  }
29
29
 
30
+ /**
31
+ * A segment's own caption box, merged over the film's (`getFootageCaptionLayout`) for the captions that start in it.
32
+ * Px of the film's frame, as `CaptionLayout`: `bottom` anchors the box by its foot (above the frame's foot) and wins;
33
+ * `top` alone hangs the box from that top.
34
+ */
35
+ export interface EdlCaption {
36
+ bottom?: number;
37
+ top?: number;
38
+ }
39
+
30
40
  export interface EdlSegment {
31
41
  /** Unique in the cut: `s<n>` (the sentence it starts with), `r<n>` (b-roll), `w<n>` (a talking clip without words). */
32
42
  id: string;
@@ -39,6 +49,8 @@ export interface EdlSegment {
39
49
  /** `kenburns`: a slow zoom over a segment without speech. */
40
50
  motion: SegmentMotion;
41
51
  audio?: EdlAudio;
52
+ /** Moves the caption box for the captions that start in this segment; absent: the film's box. */
53
+ caption?: EdlCaption;
42
54
  }
43
55
 
44
56
  export interface Edl {
@@ -75,6 +87,13 @@ export const edlSchema = z
75
87
  })
76
88
  .optional()
77
89
  .describe("Sound from another clip or span: a J- or L-cut, or b-roll over speech. Without it, the segment plays its own sound."),
90
+ caption: z
91
+ .strictObject({
92
+ bottom: z.number().min(0).optional().describe("The caption box's foot, in px above the frame's foot; anchors the box by its foot and wins over top."),
93
+ top: z.number().min(0).optional().describe("The caption box's top, in px below the frame's top; used when bottom is absent."),
94
+ })
95
+ .optional()
96
+ .describe("Moves the caption box for the captions that start in this segment (px of the film's frame). Without it, the film's caption box."),
78
97
  }),
79
98
  )
80
99
  .min(1, "the cut needs at least one segment")
@@ -221,11 +240,18 @@ export function getSoundSpan(segment: EdlSegment): { clip: string; start: number
221
240
  return segment.audio ?? { clip: segment.clip, start: segment.start, end: segment.end };
222
241
  }
223
242
 
243
+ /** The frame a segment's caption box must stay in: its height and the height of a one-line caption card, in px. */
244
+ export interface EdlCaptionBounds {
245
+ frameHeight: number;
246
+ cardHeight: number;
247
+ }
248
+
224
249
  /**
225
250
  * What makes an EDL (a hand-edited one above all) unusable with these clips; empty when it is fine: another film,
226
- * an unknown clip, a span past a clip's end, a too short segment, a repeated id, a sound span of another length.
251
+ * an unknown clip, a span past a clip's end, a too short segment, a repeated id, a sound span of another length, and
252
+ * (with `captionBounds`) a segment's caption box that would leave the frame.
227
253
  */
228
- export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[], filmId: string): string[] {
254
+ export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[], filmId: string, captionBounds?: EdlCaptionBounds): string[] {
229
255
  const problems: string[] = [];
230
256
  if (edl.film !== filmId) problems.push(`it belongs to film "${edl.film}", not "${filmId}"`);
231
257
  const durations = new Map(clips.map((clip) => [clip.id, clip.duration]));
@@ -241,6 +267,7 @@ export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[],
241
267
  seen.add(segment.id);
242
268
  checkSpan(at, segment);
243
269
  if (segment.end - segment.start < MIN_SEGMENT_SECONDS) problems.push(`${at}: shorter than ${MIN_SEGMENT_SECONDS} s (start ${segment.start}, end ${segment.end})`);
270
+ if (captionBounds !== undefined) problems.push(...findCaptionProblems(`${at}.caption`, segment.caption, captionBounds));
244
271
  const { audio } = segment;
245
272
  if (audio === undefined) return;
246
273
  checkSpan(`${at}.audio`, audio);
@@ -251,6 +278,19 @@ export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[],
251
278
  return problems;
252
279
  }
253
280
 
281
+ /** A caption box's edges past the frame: each edge at most the frame's height minus one card, so a card always fits. */
282
+ function findCaptionProblems(at: string, caption: EdlCaption | undefined, bounds: EdlCaptionBounds): string[] {
283
+ const limit = Math.round(bounds.frameHeight - bounds.cardHeight);
284
+ const problems: string[] = [];
285
+ for (const edge of ["bottom", "top"] as const) {
286
+ const value = caption?.[edge];
287
+ if (value === undefined) continue;
288
+ if (value < 0) problems.push(`${at}.${edge}: ${value} px is outside the frame`);
289
+ else if (value > limit) problems.push(`${at}.${edge}: ${value} px leaves no room for a caption card (at most ${limit} px in a ${bounds.frameHeight} px frame)`);
290
+ }
291
+ return problems;
292
+ }
293
+
254
294
  /** `validateEdl` under its first name. */
255
295
  export const findEdlProblems = validateEdl;
256
296
 
@@ -326,6 +366,11 @@ export function diffEdl(before: Edl, after: Edl): string[] {
326
366
  if (a.transition !== b.transition) lines.push(`${id}: ${a.transition} → ${b.transition}`);
327
367
  if (a.motion !== b.motion) lines.push(`${id}: motion ${a.motion} → ${b.motion}`);
328
368
  }
369
+ for (const segment of after.segments) {
370
+ const previous = old.get(segment.id);
371
+ if (previous === undefined || isSameCaption(previous.caption, segment.caption)) continue;
372
+ lines.push(isMovedCaption(segment.caption) ? `${segment.id} captions moved` : `${segment.id} captions back in the film's box`);
373
+ }
329
374
  for (const segment of after.segments) {
330
375
  const was = old.get(segment.id)?.audio;
331
376
  const { audio } = segment;
@@ -341,10 +386,32 @@ export function diffEdl(before: Edl, after: Edl): string[] {
341
386
  return lines;
342
387
  }
343
388
 
389
+ /** Whether a segment's caption field moves its box (an empty object does not). */
390
+ function isMovedCaption(caption: EdlCaption | undefined): boolean {
391
+ return caption?.bottom !== undefined || caption?.top !== undefined;
392
+ }
393
+
394
+ function isSameCaption(a: EdlCaption | undefined, b: EdlCaption | undefined): boolean {
395
+ return a?.bottom === b?.bottom && a?.top === b?.top;
396
+ }
397
+
344
398
  export function getEdlDuration(edl: Edl): number {
345
399
  return r3(edl.segments.reduce((sum, segment) => sum + (segment.end - segment.start), 0));
346
400
  }
347
401
 
402
+ /**
403
+ * The index of the segment that plays at `time` (film seconds): the one whose span holds it; a time before the film
404
+ * gives the first segment, one at or past the cut's end the last.
405
+ */
406
+ export function findSegmentAt(edl: Edl, time: number): number {
407
+ const starts = getSegmentStarts(edl);
408
+ let found = 0;
409
+ starts.forEach((start, index) => {
410
+ if (start <= time) found = index;
411
+ });
412
+ return found;
413
+ }
414
+
348
415
  /** Where each segment starts in the film, in seconds. */
349
416
  export function getSegmentStarts(edl: Edl): number[] {
350
417
  const starts: number[] = [];
@@ -1,6 +1,7 @@
1
1
  import type { VideoFormat } from "../compose/timeline.js";
2
2
  import type { EndCard } from "../film.js";
3
- import type { LoudnessTarget } from "./loudness.js";
3
+ import type { DeclickLevel } from "./declick.js";
4
+ import type { LoudnessTarget, MixLoudnessTarget } from "./loudness.js";
4
5
 
5
6
  /**
6
7
  * A footage film: filmed clips (a person talking to camera, b-roll) cut by their transcript into a captioned
@@ -131,18 +132,28 @@ export interface FootageFilm {
131
132
  * `CAPTION_MIN_FONT_SCALE` of the box's size) until it fits. Absent: no limit. The draft keeps its font size.
132
133
  */
133
134
  captionMaxLines?: number;
135
+ /**
136
+ * Words a caption card must not end on: the card hands them to the next one, so `w` stays with `domu`. Compared
137
+ * lower-case; a number always keeps with the word after it. Absent: the language's `DEFAULT_CAPTION_KEEP_WITH_NEXT`;
138
+ * empty: numbers only.
139
+ */
140
+ captionKeepWithNext?: string[];
134
141
  /** The cut's look: zoom size, fade length, Ken Burns, the end card's whoosh. Absent fields take `DEFAULT_EDIT`. */
135
142
  edit?: Partial<FootageEdit>;
136
143
  /** The closing card after the last segment; null: the film ends with its last segment. */
137
144
  endCard: EndCard | null;
145
+ /** The speech track's cleanup before its loudness normalization. Absent: none. */
146
+ speech?: FootageSpeech;
138
147
  /** The speech track's loudness, normalized before the music is mixed under it. Absent: −14 LUFS, −1 dBTP; null: left as filmed. */
139
148
  loudness?: LoudnessTarget | null;
140
149
  /**
141
150
  * The finished film's mix (speech, music, sound effects), measured after the render and normalized in two passes of
142
151
  * `loudnorm` (one linear gain; the picture is copied, only the sound re-encoded), so the published file plays at the
143
- * target. A silent mix is left as it is. Absent: −14 LUFS, −1 dBTP; null: the mix as rendered.
152
+ * target. A silent mix is left as it is. Absent: −14 LUFS, −1 dBTP; null: the mix as rendered. `keep: "speech"`
153
+ * applies no gain while the mix measures inside `target ± tolerance` (default 1 LU), so the speech keeps its level
154
+ * whatever the music volume (see `MixLoudnessTarget`).
144
155
  */
145
- mixLoudness?: LoudnessTarget | null;
156
+ mixLoudness?: MixLoudnessTarget | null;
146
157
  /**
147
158
  * The cover JPG written next to the film. Absent or without `at`: the middle of the first talking segment; `at`:
148
159
  * that second of the film; null: no cover.
@@ -150,12 +161,35 @@ export interface FootageFilm {
150
161
  cover?: { at?: number } | null;
151
162
  }
152
163
 
164
+ /** Settings of the speech track itself. */
165
+ export interface FootageSpeech {
166
+ /**
167
+ * Mouth clicks and lip smacks between words (issue #1084): `gentle` lowers the pauses by 15 dB and removes impulses
168
+ * that jump 15 dB above the band's median, `strong` by 20 dB and 9 dB. Absent or `off`: no pass, the track unchanged.
169
+ */
170
+ declick?: DeclickLevel;
171
+ }
172
+
173
+ /** The film's declick level; `off` when it sets none. */
174
+ export function getDeclickLevel(film: Pick<FootageFilm, "speech">): DeclickLevel {
175
+ return film.speech?.declick ?? "off";
176
+ }
177
+
153
178
  /** Filler words per language, written with repeated letters collapsed (`yyy` and `yyyy` are both `y`). */
154
179
  export const DEFAULT_FILLERS: Record<string, string[]> = {
155
180
  pl: ["y", "e", "yhm", "yh", "ym", "em", "hm", "mhm", "eh", "uh"],
156
181
  en: ["um", "uh", "uhm", "er", "erm", "hm", "mhm", "ah"],
157
182
  };
158
183
 
184
+ /**
185
+ * Words a caption card does not end on, per language: Polish one-letter words and short prepositions, English
186
+ * articles. A language without a list keeps numbers only.
187
+ */
188
+ export const DEFAULT_CAPTION_KEEP_WITH_NEXT: Record<string, string[]> = {
189
+ pl: ["a", "i", "o", "u", "w", "z", "we", "ze", "na", "do", "od", "po", "za", "ku"],
190
+ en: ["a", "an", "the"],
191
+ };
192
+
159
193
  /** The default fillers of a language; none for a language without a list. */
160
194
  export function getDefaultFillers(language: string): string[] {
161
195
  return DEFAULT_FILLERS[language] ?? [];
@@ -0,0 +1,161 @@
1
+ import { existsSync, mkdirSync } from "node:fs";
2
+ import { join } from "node:path";
3
+
4
+ import { err, ok } from "../voice/provider.js";
5
+ import { probeAudio } from "./audio.js";
6
+ import { writeCacheFile } from "./cache.js";
7
+ import type { Edl } from "./edl.js";
8
+ import type { FootageFilm } from "./film.js";
9
+ import { getClipCacheKey } from "./paid.js";
10
+ import { getFfmpegReporting, type FootageResult, type FootageRunOptions, type PreparedClip } from "./pipeline.js";
11
+ import { runFfmpeg, withFfmpegFailureReporting } from "./process.js";
12
+ import { toPaidAudio } from "./providers.js";
13
+ import {
14
+ buildRangeAudioArgs,
15
+ buildRangeLayArgs,
16
+ getSpeechRangeCache,
17
+ getSpeechRangeWindows,
18
+ planSpeechRanges,
19
+ type LaidRange,
20
+ type SpeechRange,
21
+ } from "./speech-ranges.js";
22
+
23
+ /**
24
+ * Voice isolation of only what the cut hears (issue #1087): each talking clip's sound ranges (`planSpeechRanges`) are
25
+ * cleaned one by one and cached by the clip's identity, the range in milliseconds and the provider, so an unchanged
26
+ * segment is never paid twice and a changed cut pays only its new ranges. The cleaned ranges are laid back over the
27
+ * clip's own sound into one file as long as the clip: the speech map `renderFootage` takes.
28
+ */
29
+
30
+ export type IsolateSpeechRangesOptions = Pick<FootageRunOptions, "buildDir" | "cacheDir" | "providers" | "isCommit" | "signal" | "onCharge" | "onProgress" | "onFfmpegFailure"> & {
31
+ /** Seconds added on each side of a sound span; default `DEFAULT_SPEECH_RANGE_MARGIN` (0.25). */
32
+ margin?: number;
33
+ log?: (line: string) => void;
34
+ };
35
+
36
+ export interface IsolatedSpeechRange extends SpeechRange {
37
+ /** The paid cache's key and file of the range's cleaned voice. */
38
+ cacheKey: string;
39
+ path: string;
40
+ /** Whether the cleaned voice was in the cache before this call. */
41
+ cached: boolean;
42
+ /** Whether it is laid in the clip's speech file (cached, or bought with `isCommit`). */
43
+ laid: boolean;
44
+ }
45
+
46
+ export interface IsolatedSpeech {
47
+ /** Every clip's audio for the speech track: its laid file, or the normalized clip when none of its ranges is laid. */
48
+ speech: Record<string, string>;
49
+ ranges: IsolatedSpeechRange[];
50
+ }
51
+
52
+ /**
53
+ * Cleans the voice of the ranges the cut hears and answers the speech map the render takes (`TranscribeOutcome.speech`).
54
+ * With `isCommit` every uncached range is bought (`onCharge` step `isolate`, item the clip id, with its `range`);
55
+ * without it only cached ranges are laid, a free draft, and `ranges` names what is still missing. Each clip with a laid
56
+ * range gets `<buildDir>/audio/<id>.ranges.mp3`.
57
+ */
58
+ export function isolateSpeechRanges(film: FootageFilm, clips: PreparedClip[], edl: Edl, options: IsolateSpeechRangesOptions): Promise<FootageResult<IsolatedSpeech>> {
59
+ const log = options.log ?? ((): void => undefined);
60
+ return withFfmpegFailureReporting(getFfmpegReporting({ log, ...(options.onFfmpegFailure === undefined ? {} : { onFfmpegFailure: options.onFfmpegFailure }) }), () =>
61
+ runIsolateSpeechRanges(film, clips, edl, options, log),
62
+ );
63
+ }
64
+
65
+ /** The body of `isolateSpeechRanges`, inside its ffmpeg failure reporting. */
66
+ async function runIsolateSpeechRanges(
67
+ film: FootageFilm,
68
+ clips: PreparedClip[],
69
+ edl: Edl,
70
+ options: IsolateSpeechRangesOptions,
71
+ log: (line: string) => void,
72
+ ): Promise<FootageResult<IsolatedSpeech>> {
73
+ const { buildDir, providers, signal } = options;
74
+ const aborting = signal === undefined ? {} : { signal };
75
+ const planned = planSpeechRanges(
76
+ edl,
77
+ clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration })),
78
+ options.margin === undefined ? {} : { margin: options.margin },
79
+ );
80
+ const byClip = new Map(clips.map((clip) => [clip.clip.id, clip]));
81
+ const clipKeys = new Map<string, string>();
82
+ const ranges: IsolatedSpeechRange[] = [];
83
+ for (const range of planned) {
84
+ const clip = byClip.get(range.clip);
85
+ if (clip === undefined) continue;
86
+ let clipKey = clipKeys.get(range.clip);
87
+ if (clipKey === undefined) {
88
+ clipKey = await getClipCacheKey(clip.source, clip.clip);
89
+ clipKeys.set(range.clip, clipKey);
90
+ }
91
+ const cache = getSpeechRangeCache(film.id, clipKey, range, { cacheDir: options.cacheDir, provider: providers.isolation.id });
92
+ const cached = existsSync(cache.path);
93
+ ranges.push({ ...range, cacheKey: cache.key, path: cache.path, cached, laid: cached });
94
+ }
95
+ const buying = options.isCommit ? ranges.filter((range) => !range.cached) : [];
96
+ const total = buying.reduce((sum, range) => sum + (range.endMs - range.startMs) / 1000, 0);
97
+ let done = 0;
98
+ const report = (range: SpeechRange, index: number): void => {
99
+ options.onProgress?.({ step: "isolate", item: range.clip, itemIndex: index + 1, itemCount: buying.length, done: Math.round(done * 1000) / 1000, total: Math.round(total * 1000) / 1000 });
100
+ };
101
+ const workDir = join(buildDir, "audio", "ranges");
102
+ for (const [index, range] of buying.entries()) {
103
+ const clip = byClip.get(range.clip);
104
+ if (clip === undefined) continue;
105
+ const units = (range.endMs - range.startMs) / 1000;
106
+ report(range, index);
107
+ mkdirSync(workDir, { recursive: true });
108
+ // The provider sees the clip's id and the range as the file name.
109
+ const input = join(workDir, `${range.clip}.${String(range.startMs)}-${String(range.endMs)}.mp3`);
110
+ await runFfmpeg(`Cutting ${describeRange(range)} for voice isolation`, buildRangeAudioArgs(clip.file, range, input), { ...aborting, output: input });
111
+ signal?.throwIfAborted();
112
+ log(`voice isolation (${providers.isolation.id}): ${describeRange(range)}, ${units.toFixed(1)} s.`);
113
+ const cleaned = await providers.isolation.isolate({ audio: input, ...aborting });
114
+ signal?.throwIfAborted();
115
+ if (!cleaned.ok) return err(`${describeRange(range)}: ${cleaned.error}`);
116
+ const paid = toPaidAudio(cleaned.value);
117
+ writeCacheFile(range.path, paid.audio);
118
+ range.laid = true;
119
+ options.onCharge?.({
120
+ step: "isolate",
121
+ item: range.clip,
122
+ provider: providers.isolation.id,
123
+ units,
124
+ ...(paid.credits === undefined ? {} : { credits: paid.credits }),
125
+ cacheKey: range.cacheKey,
126
+ range: { start: range.startMs / 1000, end: range.endMs / 1000 },
127
+ });
128
+ log(`voice isolation: saved ${range.path} (commit it).`);
129
+ done += units;
130
+ report(range, index);
131
+ }
132
+ const missing = ranges.filter((range) => !range.laid);
133
+ if (missing.length > 0) {
134
+ const seconds = missing.reduce((sum, range) => sum + (range.endMs - range.startMs) / 1000, 0);
135
+ log(`voice isolation DRY RUN: ${String(missing.length)} range${missing.length === 1 ? "" : "s"} (${seconds.toFixed(1)} s) not cached, laid with the clip's own sound. Run with --commit to pay.`);
136
+ }
137
+ const speech: Record<string, string> = {};
138
+ for (const clip of clips) {
139
+ const id = clip.clip.id;
140
+ speech[id] = clip.file;
141
+ const laidRanges = ranges.filter((range) => range.clip === id && range.laid);
142
+ if (laidRanges.length === 0) continue;
143
+ const laid: LaidRange[] = [];
144
+ for (const window of getSpeechRangeWindows(laidRanges)) {
145
+ const file = laidRanges.find((range) => range === window.range)?.path;
146
+ if (file === undefined) continue;
147
+ const probed = await probeAudio(file, aborting);
148
+ if (!probed.ok) return err(`${describeRange(window.range)}: the cleaned voice at ${file} does not read (${probed.error}); delete it and run again.`);
149
+ laid.push({ window, file, isMono: probed.probe.channels === 1 });
150
+ }
151
+ const output = join(buildDir, "audio", `${id}.ranges.mp3`);
152
+ mkdirSync(join(buildDir, "audio"), { recursive: true });
153
+ await runFfmpeg(`Laying the cleaned voice over clip ${id}`, buildRangeLayArgs(clip.file, laid, output), { ...aborting, output });
154
+ speech[id] = output;
155
+ }
156
+ return ok({ speech, ranges });
157
+ }
158
+
159
+ function describeRange(range: SpeechRange): string {
160
+ return `clip ${range.clip} ${(range.startMs / 1000).toFixed(3)}-${(range.endMs / 1000).toFixed(3)} s`;
161
+ }