@softure-ai/marketing-kit 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/CHANGELOG.md +71 -0
  2. package/README.md +66 -9
  3. package/dist/cli/footage.js +1 -1
  4. package/dist/cli/footage.js.map +1 -1
  5. package/dist/compose/timeline.d.ts +23 -3
  6. package/dist/compose/timeline.d.ts.map +1 -1
  7. package/dist/compose/timeline.js +31 -11
  8. package/dist/compose/timeline.js.map +1 -1
  9. package/dist/config/actions-schema.d.ts +14 -14
  10. package/dist/config/config.d.ts.map +1 -1
  11. package/dist/config/config.js +2 -0
  12. package/dist/config/config.js.map +1 -1
  13. package/dist/config/schema.d.ts +83 -70
  14. package/dist/config/schema.d.ts.map +1 -1
  15. package/dist/config/schema.js +21 -1
  16. package/dist/config/schema.js.map +1 -1
  17. package/dist/config/shot-steps.d.ts +16 -16
  18. package/dist/footage/captions.d.ts +45 -3
  19. package/dist/footage/captions.d.ts.map +1 -1
  20. package/dist/footage/captions.js +65 -5
  21. package/dist/footage/captions.js.map +1 -1
  22. package/dist/footage/compose.d.ts.map +1 -1
  23. package/dist/footage/compose.js +8 -4
  24. package/dist/footage/compose.js.map +1 -1
  25. package/dist/footage/declick.d.ts +106 -0
  26. package/dist/footage/declick.d.ts.map +1 -0
  27. package/dist/footage/declick.js +296 -0
  28. package/dist/footage/declick.js.map +1 -0
  29. package/dist/footage/draft.d.ts +8 -3
  30. package/dist/footage/draft.d.ts.map +1 -1
  31. package/dist/footage/draft.js +55 -22
  32. package/dist/footage/draft.js.map +1 -1
  33. package/dist/footage/edl.d.ts +28 -2
  34. package/dist/footage/edl.d.ts.map +1 -1
  35. package/dist/footage/edl.js +53 -2
  36. package/dist/footage/edl.js.map +1 -1
  37. package/dist/footage/film.d.ts +29 -3
  38. package/dist/footage/film.d.ts.map +1 -1
  39. package/dist/footage/film.js +12 -0
  40. package/dist/footage/film.js.map +1 -1
  41. package/dist/footage/isolate-ranges.d.ts +37 -0
  42. package/dist/footage/isolate-ranges.d.ts.map +1 -0
  43. package/dist/footage/isolate-ranges.js +113 -0
  44. package/dist/footage/isolate-ranges.js.map +1 -0
  45. package/dist/footage/loudness.d.ts +58 -0
  46. package/dist/footage/loudness.d.ts.map +1 -1
  47. package/dist/footage/loudness.js +80 -0
  48. package/dist/footage/loudness.js.map +1 -1
  49. package/dist/footage/paid.d.ts +18 -0
  50. package/dist/footage/paid.d.ts.map +1 -1
  51. package/dist/footage/paid.js +32 -2
  52. package/dist/footage/paid.js.map +1 -1
  53. package/dist/footage/pipeline.d.ts +43 -5
  54. package/dist/footage/pipeline.d.ts.map +1 -1
  55. package/dist/footage/pipeline.js +103 -20
  56. package/dist/footage/pipeline.js.map +1 -1
  57. package/dist/footage/providers.d.ts +5 -0
  58. package/dist/footage/providers.d.ts.map +1 -1
  59. package/dist/footage/publish.d.ts +11 -2
  60. package/dist/footage/publish.d.ts.map +1 -1
  61. package/dist/footage/publish.js +14 -10
  62. package/dist/footage/publish.js.map +1 -1
  63. package/dist/footage/speech-ranges.d.ts +65 -0
  64. package/dist/footage/speech-ranges.d.ts.map +1 -0
  65. package/dist/footage/speech-ranges.js +98 -0
  66. package/dist/footage/speech-ranges.js.map +1 -0
  67. package/dist/index.d.ts +10 -7
  68. package/dist/index.d.ts.map +1 -1
  69. package/dist/index.js +10 -7
  70. package/dist/index.js.map +1 -1
  71. package/package.json +1 -1
  72. package/schema/edl.schema.json +17 -0
  73. package/schema/marketing.schema.json +43 -1
  74. package/src/cli/footage.ts +1 -1
  75. package/src/compose/timeline.ts +41 -10
  76. package/src/config/config.ts +2 -0
  77. package/src/config/schema.ts +23 -1
  78. package/src/footage/captions.ts +81 -6
  79. package/src/footage/compose.ts +8 -4
  80. package/src/footage/declick.ts +353 -0
  81. package/src/footage/draft.ts +58 -23
  82. package/src/footage/edl.ts +69 -2
  83. package/src/footage/film.ts +37 -3
  84. package/src/footage/isolate-ranges.ts +161 -0
  85. package/src/footage/loudness.ts +112 -0
  86. package/src/footage/paid.ts +40 -2
  87. package/src/footage/pipeline.ts +155 -22
  88. package/src/footage/providers.ts +2 -0
  89. package/src/footage/publish.ts +20 -11
  90. package/src/footage/speech-ranges.ts +131 -0
  91. package/src/index.ts +70 -0
@@ -318,25 +318,56 @@ export interface CaptionChunk {
318
318
  end: number;
319
319
  }
320
320
 
321
+ /** How words are cut into caption chunks. */
322
+ export interface CaptionChunkOptions {
323
+ /** The most words a chunk holds; default 4. */
324
+ maxWords?: number;
325
+ /**
326
+ * Words that must not end a chunk (a one-letter word, a short preposition, a number before its unit): a full chunk
327
+ * ending on one hands it, with the kept words before it, to the next chunk. A word ending in punctuation still ends
328
+ * its chunk. Absent: every word may end one.
329
+ */
330
+ keepsWithNext?: (word: TimedWord) => boolean;
331
+ }
332
+
333
+ /**
334
+ * A full chunk cut after its last word that does not keep with the next: the chunk and the words it carries over to
335
+ * the next one. A chunk of kept words only stays whole.
336
+ */
337
+ export function splitAfterKept(words: TimedWord[], keepsWithNext: ((word: TimedWord) => boolean) | undefined): { chunk: TimedWord[]; carry: TimedWord[] } {
338
+ if (keepsWithNext === undefined) return { chunk: words, carry: [] };
339
+ const lastFree = words.findLastIndex((word) => !keepsWithNext(word));
340
+ if (lastFree === -1) return { chunk: words, carry: [] };
341
+ return { chunk: words.slice(0, lastFree + 1), carry: words.slice(lastFree + 1) };
342
+ }
343
+
321
344
  /**
322
- * Words → caption chunks: at most `maxWords` words, broken after the end of a sentence and after a
323
- * comma, so a caption does not glue two thoughts together.
345
+ * Words → caption chunks: at most `maxWords` words, broken after the end of a sentence and after a comma, so a caption
346
+ * does not glue two thoughts together; never after a word `keepsWithNext` keeps, unless punctuation ends it or every
347
+ * word of the chunk is kept. A number as `options` is `maxWords`.
324
348
  */
325
- export function captionChunks(words: TimedWord[], maxWords = 4): CaptionChunk[] {
349
+ export function captionChunks(words: TimedWord[], options: number | CaptionChunkOptions = {}): CaptionChunk[] {
350
+ const { maxWords = 4, keepsWithNext } = typeof options === "number" ? { maxWords: options } : options;
326
351
  const chunks: CaptionChunk[] = [];
327
352
  let current: TimedWord[] = [];
328
- const flush = () => {
329
- const first = current[0];
330
- const last = current[current.length - 1];
353
+ const push = (chunkWords: TimedWord[]) => {
354
+ const first = chunkWords[0];
355
+ const last = chunkWords[chunkWords.length - 1];
331
356
  if (first === undefined || last === undefined) return;
332
- chunks.push({ words: current, start: first.start, end: last.end });
333
- current = [];
357
+ chunks.push({ words: chunkWords, start: first.start, end: last.end });
334
358
  };
335
359
  for (const word of words) {
336
360
  current.push(word);
337
- if (/[.,?!:;]$/.test(word.text) || current.length === maxWords) flush();
361
+ if (/[.,?!:;]$/.test(word.text)) {
362
+ push(current);
363
+ current = [];
364
+ } else if (current.length >= maxWords) {
365
+ const { chunk, carry } = splitAfterKept(current, keepsWithNext);
366
+ push(chunk);
367
+ current = carry;
368
+ }
338
369
  }
339
- flush();
370
+ push(current);
340
371
  return chunks;
341
372
  }
342
373
 
@@ -247,6 +247,7 @@ function resolveFootage(data: MarketingJson, at: (relative: string) => string):
247
247
  captionStyle: film.captionStyle,
248
248
  ...(film.captionPreset === undefined ? {} : { captionPreset: film.captionPreset }),
249
249
  ...(film.captionMaxLines === undefined ? {} : { captionMaxLines: film.captionMaxLines }),
250
+ ...(film.captionKeepWithNext === undefined ? {} : { captionKeepWithNext: film.captionKeepWithNext }),
250
251
  edit: {
251
252
  zoomPercent: film.edit.zoomPercent,
252
253
  zoomEaseFrames: film.edit.zoomEaseFrames,
@@ -257,6 +258,7 @@ function resolveFootage(data: MarketingJson, at: (relative: string) => string):
257
258
  isCutAudioCrossfade: film.edit.cutAudioCrossfade,
258
259
  },
259
260
  endCard: film.endCard ?? null,
261
+ ...(film.speech.declick === "off" ? {} : { speech: { declick: film.speech.declick } }),
260
262
  loudness: film.loudness === false ? null : film.loudness,
261
263
  mixLoudness: film.mixLoudness === false ? null : film.mixLoudness,
262
264
  cover: film.cover === false ? null : film.cover,
@@ -14,6 +14,7 @@ import {
14
14
  } from "../compose/timeline.js";
15
15
  import { bigNumberDataSchema, carouselDataSchema, headlineChartDataSchema, headlineCtaDataSchema } from "../og/templates/schemas.js";
16
16
  import { CHANNEL_CODE_MAX_LENGTH, CHANNEL_CODE_PATTERN, DEFAULT_LINK_IN_BIO, PLATFORMS } from "../platforms.js";
17
+ import { DECLICK_LEVELS } from "../footage/declick.js";
17
18
  import { CAPTION_PRESETS, CAPTION_STYLES, CLIP_FITS, CLIP_ROLES, CUT_TRANSITIONS, JUMP_TRANSITIONS } from "../footage/film.js";
18
19
  import { normalizeKeyTerms } from "../footage/keyterms.js";
19
20
  import { ELEVENLABS_DEFAULT_MODEL } from "../voice/voiceover.js";
@@ -574,6 +575,11 @@ const footageSchema = z
574
575
  .max(4)
575
576
  .optional()
576
577
  .describe("The most lines a caption card may take (1-4): the render shrinks a longer card's font, to 70 % at most; without it, no limit."),
578
+ captionKeepWithNext: z
579
+ .array(z.string().trim().min(1).max(20))
580
+ .max(100)
581
+ .optional()
582
+ .describe("Words a caption card must not end on (they move to the next card), compared lower-case; a number always keeps with the next word. Without it, the language's default (pl: one-letter words and short prepositions, en: articles); [] keeps numbers only."),
577
583
  edit: z
578
584
  .strictObject({
579
585
  zoomPercent: z.number().min(105).max(112).default(108).describe("How far a zoom cut punches in (105-112 %, default 108)."),
@@ -596,6 +602,17 @@ const footageSchema = z
596
602
  .prefault({})
597
603
  .describe("The cut's look; the defaults read as an ordinary phone video."),
598
604
  endCard: endCardSchema.optional().describe("The closing card after the last segment; without it, the film ends with its last segment."),
605
+ speech: z
606
+ .strictObject({
607
+ declick: z
608
+ .enum(DECLICK_LEVELS)
609
+ .default("off")
610
+ .describe(
611
+ "Mouth clicks and lip smacks between words, cleaned before the loudness pass: gentle lowers the pauses by 15 dB and removes short clicks, strong by 20 dB with a more sensitive detection; off (the default) leaves the speech track as cut.",
612
+ ),
613
+ })
614
+ .prefault({})
615
+ .describe("The speech track's cleanup before its loudness normalization."),
599
616
  loudness: z
600
617
  .union([
601
618
  z.literal(false),
@@ -612,10 +629,15 @@ const footageSchema = z
612
629
  z.strictObject({
613
630
  target: z.number().min(-40).max(-5).default(-14).describe("Integrated loudness in LUFS (-40 to -5, default -14)."),
614
631
  truePeak: z.number().min(-9).max(0).default(-1).describe("The highest true peak in dBTP (-9 to 0, default -1)."),
632
+ tolerance: z.number().min(0.2).max(3).optional().describe("How far from target the mix may land, in LU (0.2 to 3, default 1)."),
633
+ keep: z
634
+ .enum(["loudness", "speech"])
635
+ .optional()
636
+ .describe('"loudness" (default): one gain brings the mix to target; "speech": no gain while the mix measures inside target ± tolerance, so the speech keeps its level whatever the music volume.'),
615
637
  }),
616
638
  ])
617
639
  .default({ target: -14, truePeak: -1 })
618
- .describe("The finished film's mix (speech and music), measured after the render and normalized in two passes; false leaves the mix as rendered. Default -14 LUFS, -1 dBTP."),
640
+ .describe("The finished film's mix (speech and music), measured after the render, brought to the target by one gain and held under the true peak by a limiter; false leaves the mix as rendered. Default -14 LUFS, -1 dBTP."),
619
641
  cover: z
620
642
  .union([
621
643
  z.literal(false),
@@ -1,10 +1,11 @@
1
1
  import { readFileSync } from "node:fs";
2
2
 
3
- import { resolveLayout, type CaptionLayout, type LayoutOverride } from "../compose/timeline.js";
3
+ import { CAPTION_LINE_HEIGHT, CAPTION_PILL_PADDING, captionChunks, resolveLayout, splitAfterKept, type CaptionLayout, type LayoutOverride } from "../compose/timeline.js";
4
4
  import type { BrandFont } from "../config/config.js";
5
5
  import { readCharacterMap, type CharacterMap } from "../og/character-map.js";
6
6
  import type { TimedWord } from "../voice/voiceover.js";
7
- import type { CaptionPreset, FootageFilm } from "./film.js";
7
+ import { findSegmentAt, type Edl, type EdlCaption, type EdlCaptionBounds } from "./edl.js";
8
+ import { DEFAULT_CAPTION_KEEP_WITH_NEXT, type CaptionPreset, type FootageFilm } from "./film.js";
8
9
 
9
10
  /**
10
11
  * The footage film's captions: plain phrases (the default style), the 9:16 caption box inside a platform's safe zone,
@@ -23,23 +24,39 @@ const MAX_PHRASE_WORDS = 6;
23
24
  /** A silence this long between two words starts a new phrase. */
24
25
  const PHRASE_GAP_SECONDS = 0.7;
25
26
 
27
+ /** How words are cut into phrases. */
28
+ export interface PhraseChunkOptions {
29
+ /**
30
+ * Words that must not end a phrase (`getCaptionKeepsWithNext`): neither a pause nor the six-word limit ends a
31
+ * phrase on one (a full phrase hands it to the next); punctuation still does. Absent: every word may end one.
32
+ */
33
+ keepsWithNext?: (word: TimedWord) => boolean;
34
+ }
35
+
26
36
  /**
27
- * The words as phrases of 2-6: a phrase ends at punctuation, after six words or at a pause; a single word left alone
37
+ * The words as phrases of 2-6: a phrase ends at punctuation, after six words or at a pause, never on a word
38
+ * `keepsWithNext` keeps (unless punctuation ends it or the phrase holds kept words only); a single word left alone
28
39
  * joins the phrase before it when that one has room and no pause separates them.
29
40
  */
30
- export function phraseChunks(words: TimedWord[]): PhraseChunk[] {
41
+ export function phraseChunks(words: TimedWord[], options: PhraseChunkOptions = {}): PhraseChunk[] {
42
+ const { keepsWithNext } = options;
31
43
  const groups: TimedWord[][] = [];
32
44
  let current: TimedWord[] = [];
33
45
  words.forEach((word, index) => {
34
46
  const previous = words[index - 1];
35
- if (current.length > 0 && previous !== undefined && word.start - previous.end > PHRASE_GAP_SECONDS) {
47
+ const isPreviousKept = previous !== undefined && keepsWithNext !== undefined && keepsWithNext(previous);
48
+ if (current.length > 0 && previous !== undefined && !isPreviousKept && word.start - previous.end > PHRASE_GAP_SECONDS) {
36
49
  groups.push(current);
37
50
  current = [];
38
51
  }
39
52
  current.push(word);
40
- if (/[.,?!:;…]$/u.test(word.text) || current.length === MAX_PHRASE_WORDS) {
53
+ if (/[.,?!:;…]$/u.test(word.text)) {
41
54
  groups.push(current);
42
55
  current = [];
56
+ } else if (current.length === MAX_PHRASE_WORDS) {
57
+ const { chunk, carry } = splitAfterKept(current, keepsWithNext);
58
+ groups.push(chunk);
59
+ current = carry;
43
60
  }
44
61
  });
45
62
  if (current.length > 0) groups.push(current);
@@ -55,6 +72,31 @@ export function phraseChunks(words: TimedWord[]): PhraseChunk[] {
55
72
  return merged.map((group) => ({ words: group, start: (group[0] as TimedWord).start, end: (group[group.length - 1] as TimedWord).end }));
56
73
  }
57
74
 
75
+ /** What a film's caption cards are cut by; without `language`, no language default applies. */
76
+ export type CaptionWordsFilm = Pick<FootageFilm, "captionKeepWithNext"> & { language?: string };
77
+
78
+ const NUMBER_PATTERN = /^\d+(?:[.,]\d+)*$/u;
79
+
80
+ /**
81
+ * Which of a film's words keep with the next on a caption card: a number, and a word of `captionKeepWithNext` (absent:
82
+ * the language's `DEFAULT_CAPTION_KEEP_WITH_NEXT`), compared lower-case. The render and the draft cut cards alike.
83
+ */
84
+ export function getCaptionKeepsWithNext(film: CaptionWordsFilm): (word: TimedWord) => boolean {
85
+ const defaults = film.language === undefined ? [] : (DEFAULT_CAPTION_KEEP_WITH_NEXT[film.language] ?? []);
86
+ const words = new Set((film.captionKeepWithNext ?? defaults).map((text) => text.toLowerCase()));
87
+ return (word) => NUMBER_PATTERN.test(word.text) || words.has(word.text.toLowerCase());
88
+ }
89
+
90
+ /**
91
+ * A footage film's caption cards in film time: none without captions, `captionChunks` for `highlight`, `phraseChunks`
92
+ * otherwise; both keep the film's kept words with the next (`getCaptionKeepsWithNext`). The render and the draft share it.
93
+ */
94
+ export function getFootageCaptionChunks(film: CaptionWordsFilm & Pick<FootageFilm, "captions" | "captionStyle">, words: TimedWord[]): PhraseChunk[] {
95
+ if (!film.captions) return [];
96
+ const keepsWithNext = getCaptionKeepsWithNext(film);
97
+ return film.captionStyle === "highlight" ? captionChunks(words, { keepsWithNext }) : phraseChunks(words, { keepsWithNext });
98
+ }
99
+
58
100
  /** When a caption chunk is on screen, in film seconds. */
59
101
  export interface CaptionSpan {
60
102
  start: number;
@@ -128,6 +170,39 @@ export function getFootageCaptionLayout(film: Pick<FootageFilm, "format" | "capt
128
170
  return getCaptionPresetLayout(film.captionPreset, base.caption.fontSize);
129
171
  }
130
172
 
173
+ /**
174
+ * A caption box moved by a segment's `caption` field: its `bottom` anchors the box by its foot; its `top` alone hangs
175
+ * the box from that top (the film's `bottom` dropped). Without either, the film's box itself.
176
+ */
177
+ export function applySegmentCaption(base: CaptionLayout, caption: EdlCaption | undefined): CaptionLayout {
178
+ if (caption?.bottom !== undefined) return { ...base, top: caption.top ?? base.top, bottom: caption.bottom };
179
+ if (caption?.top === undefined) return base;
180
+ return { top: caption.top, left: base.left, right: base.right, fontSize: base.fontSize };
181
+ }
182
+
183
+ /** The caption box of one segment of the cut: the film's box (`getFootageCaptionLayout`) with the segment's `caption` over it. */
184
+ export function getSegmentCaptionLayout(film: Pick<FootageFilm, "format" | "captionPreset">, edl: Edl, segmentIndex: number, override: LayoutOverride = {}): CaptionLayout {
185
+ return applySegmentCaption(getFootageCaptionLayout(film, override), edl.segments[segmentIndex]?.caption);
186
+ }
187
+
188
+ /**
189
+ * The box each caption chunk is drawn in: the box of the segment where the chunk's first word starts (film time), so
190
+ * a chunk that spans a cut keeps one box. The render and the draft place chunks alike.
191
+ */
192
+ export function getCaptionChunkLayouts(chunks: readonly { start: number }[], edl: Edl, base: CaptionLayout): CaptionLayout[] {
193
+ return chunks.map((chunk) => applySegmentCaption(base, edl.segments[findSegmentAt(edl, chunk.start)]?.caption));
194
+ }
195
+
196
+ /** Whether two caption boxes sit in the same place (a chunk in the film's box is drawn exactly as before). */
197
+ export function isSameCaptionPlace(a: CaptionLayout, b: CaptionLayout): boolean {
198
+ return a.bottom === b.bottom && (a.bottom !== undefined || a.top === b.top);
199
+ }
200
+
201
+ /** The frame a segment's caption box must stay in (`validateEdl`): the frame's height and a one-line card's. */
202
+ export function getCaptionBounds(frameHeight: number, caption: Pick<CaptionLayout, "fontSize">): EdlCaptionBounds {
203
+ return { frameHeight, cardHeight: Math.ceil(caption.fontSize * CAPTION_LINE_HEIGHT + 2 * CAPTION_PILL_PADDING.y) };
204
+ }
205
+
131
206
  /** A character a font is asked to draw and does not have. */
132
207
  export interface MissingCaptionGlyphs {
133
208
  /** The family, as `brand.fonts` names it. */
@@ -2,9 +2,9 @@ import type { BrandColors } from "../config/colors.js";
2
2
  import type { SfxEvent } from "../config/schema.js";
3
3
  import type { ComposeFont } from "../compose/compose.js";
4
4
  import { escapeHtml, fontFaces, fontStack } from "../compose/html.js";
5
- import { CAPTION_LINE_HEIGHT, captionChunks, getCaptionPillCss, getCaptionPositionCss, type CaptionLayout } from "../compose/timeline.js";
5
+ import { CAPTION_LINE_HEIGHT, getCaptionPillCss, getCaptionPositionCss, type CaptionLayout } from "../compose/timeline.js";
6
6
  import type { TimedWord } from "../voice/voiceover.js";
7
- import { getCaptionSpans, phraseChunks, type CaptionSpan } from "./captions.js";
7
+ import { getCaptionChunkLayouts, getCaptionSpans, getFootageCaptionChunks, isSameCaptionPlace, type CaptionSpan } from "./captions.js";
8
8
  import { getEdlDuration, getSegmentStarts, type Edl } from "./edl.js";
9
9
  import { CAPTION_MIN_FONT_SCALE, DEFAULT_EDIT, getFootageEdit, type FootageEdit, type FootageFilm } from "./film.js";
10
10
  import { FOOTAGE_FPS } from "./normalize.js";
@@ -142,15 +142,19 @@ export function composeFootage(input: FootageComposeInput): string {
142
142
 
143
143
  // --- Captions: plain phrases, or the pill that lights each word ---
144
144
  const isHighlight = film.captionStyle === "highlight";
145
- const caption = !film.captions ? [] : isHighlight ? captionChunks(words) : phraseChunks(words);
145
+ const caption = getFootageCaptionChunks(film, words);
146
146
  const spans = getCaptionSpans(caption, cutEnd);
147
+ const boxes = getCaptionChunkLayouts(caption, edl, captionBox);
147
148
  const captionHtml = caption
148
149
  .map((chunk, i) => {
149
150
  const { start, end: stop } = spans[i] as CaptionSpan;
151
+ const place = boxes[i] ?? captionBox;
152
+ // A chunk in a segment's own box overrides the shared rule's anchor; one in the film's box is written as before.
153
+ const style = isSameCaptionPlace(place, captionBox) ? "" : ` style="${place.bottom === undefined ? `bottom:auto;top:${place.top}px` : `top:auto;bottom:${place.bottom}px`}"`;
150
154
  const box = isHighlight
151
155
  ? `<p class="pill">${chunk.words.map((w, j) => `<span id="cw${i}_${j}">${escapeHtml(w.text)}</span>`).join(" ")}</p>`
152
156
  : `<p class="phrase">${escapeHtml(chunk.words.map((w) => w.text).join(" "))}</p>`;
153
- return `<div id="cap${i}" class="clip caption" data-start="${start}" data-duration="${r3(stop - start)}">${box}</div>`;
157
+ return `<div id="cap${i}" class="clip caption" data-start="${start}" data-duration="${r3(stop - start)}"${style}>${box}</div>`;
154
158
  })
155
159
  .join("\n ");
156
160
 
@@ -0,0 +1,353 @@
1
+ import { readFileSync, rmSync, writeFileSync } from "node:fs";
2
+ import { extname } from "node:path";
3
+
4
+ import type { TimedWord } from "../voice/voiceover.js";
5
+ import { FOOTAGE_SAMPLE_RATE } from "./normalize.js";
6
+ import { runFfmpeg } from "./process.js";
7
+
8
+ /**
9
+ * The speech track's declick stage (issue #1084), between the cut speech track and its loudness normalization: mouth
10
+ * clicks and lip smacks between words are short broadband bursts well below the speech peaks, which ffmpeg's
11
+ * `adeclick` does not see. Two steps over the cut speech, both from the words' film times:
12
+ *
13
+ * - a gate: outside each word's window (start and end ± 40 ms) the speech is lowered, with a 20 ms linear fade that
14
+ * sits outside the window, so no word onset or tail is touched;
15
+ * - impulse removal: outside the words' own spans, bursts under 30 ms that jump above the 2-8 kHz band's running median
16
+ * are lowered by 30 dB with a short crossfade on both sides.
17
+ *
18
+ * The gain is applied per sample to the decoded PCM (a `volume` expression is evaluated only once per audio frame,
19
+ * about 21 ms, coarser than the crossfades).
20
+ */
21
+
22
+ export const DECLICK_LEVELS = ["off", "gentle", "strong"] as const;
23
+
24
+ export type DeclickLevel = (typeof DECLICK_LEVELS)[number];
25
+
26
+ /** A level that runs the stage. */
27
+ export type ActiveDeclickLevel = Exclude<DeclickLevel, "off">;
28
+
29
+ export interface DeclickSettings {
30
+ /** How far the speech is lowered outside the words' windows, in dB (negative). */
31
+ gateDb: number;
32
+ /** How far a frame's 2-8 kHz energy must rise above the band's running median to count as an impulse, in dB. */
33
+ jumpDb: number;
34
+ /** The crossfade on both sides of a lowered impulse, in seconds. */
35
+ crossfadeSeconds: number;
36
+ }
37
+
38
+ export const DECLICK_SETTINGS: Record<ActiveDeclickLevel, DeclickSettings> = {
39
+ gentle: { gateDb: -15, jumpDb: 15, crossfadeSeconds: 0.005 },
40
+ strong: { gateDb: -20, jumpDb: 9, crossfadeSeconds: 0.008 },
41
+ };
42
+
43
+ /** Seconds a word's window reaches past its start and end; the gate leaves the window untouched. */
44
+ export const DECLICK_WORD_WINDOW = 0.04;
45
+ /** The gate's fade, outside the window. */
46
+ export const DECLICK_GATE_FADE = 0.02;
47
+ /** The longest burst that counts as an impulse. */
48
+ export const DECLICK_MAX_IMPULSE = 0.03;
49
+ /** How far an impulse is lowered, in dB. */
50
+ export const DECLICK_IMPULSE_DB = -30;
51
+
52
+ const BAND_LOW_HZ = 2000;
53
+ const BAND_HIGH_HZ = 8000;
54
+ const FRAME_SECONDS = 0.002;
55
+ const HOP_SECONDS = 0.001;
56
+ /** The running median's reach on each side of a frame. */
57
+ const MEDIAN_SECONDS = 0.1;
58
+ /** A lowered impulse reaches this far past its detected frames, for the band filter's delay and the frames' edges. */
59
+ const IMPULSE_MARGIN = 0.001;
60
+ /** A frame quieter than this (mean square, dB) is never an impulse: digital silence with a stray bit is not a click. */
61
+ const BAND_FLOOR_DB = -90;
62
+
63
+ export interface TimeSpan {
64
+ start: number;
65
+ end: number;
66
+ }
67
+
68
+ /** What the stage did, in the render's and the draft's ok value. */
69
+ export interface DeclickReport {
70
+ level: ActiveDeclickLevel;
71
+ /** Impulses lowered. */
72
+ impulses: number;
73
+ /** The gate between words in dB; null when the track has no words to tell speech from pauses (no gate). */
74
+ gateDb: number | null;
75
+ }
76
+
77
+ const fromDb = (db: number): number => 10 ** (db / 20);
78
+
79
+ const sortSpans = (words: readonly TimeSpan[]): TimeSpan[] => [...words].filter((word) => word.end > word.start).sort((a, b) => a.start - b.start);
80
+
81
+ /**
82
+ * The windows the gate leaves open: each word's start and end ± `DECLICK_WORD_WINDOW`, merged when two windows (with
83
+ * their fades) would touch, so the gain never dips between two close words.
84
+ */
85
+ export function getGateWindows(words: readonly TimeSpan[]): TimeSpan[] {
86
+ const windows: TimeSpan[] = [];
87
+ for (const word of sortSpans(words)) {
88
+ const start = Math.max(0, word.start - DECLICK_WORD_WINDOW);
89
+ const end = word.end + DECLICK_WORD_WINDOW;
90
+ const last = windows.at(-1);
91
+ if (last !== undefined && start - last.end < 2 * DECLICK_GATE_FADE) last.end = Math.max(last.end, end);
92
+ else windows.push({ start, end });
93
+ }
94
+ return windows;
95
+ }
96
+
97
+ /** One second-order section (RBJ cookbook), run in place over the samples. */
98
+ function runBiquad(samples: Float64Array, kind: "highpass" | "lowpass", frequency: number, sampleRate: number): void {
99
+ const w0 = (2 * Math.PI * Math.min(frequency, sampleRate * 0.45)) / sampleRate;
100
+ const cos = Math.cos(w0);
101
+ const alpha = Math.sin(w0) / (2 * Math.SQRT1_2);
102
+ const a0 = 1 + alpha;
103
+ const b0 = (kind === "lowpass" ? (1 - cos) / 2 : (1 + cos) / 2) / a0;
104
+ const b1 = (kind === "lowpass" ? 1 - cos : -(1 + cos)) / a0;
105
+ const b2 = b0;
106
+ const a1 = (-2 * cos) / a0;
107
+ const a2 = (1 - alpha) / a0;
108
+ let x1 = 0;
109
+ let x2 = 0;
110
+ let y1 = 0;
111
+ let y2 = 0;
112
+ for (let index = 0; index < samples.length; index++) {
113
+ const x = samples[index] ?? 0;
114
+ const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
115
+ x2 = x1;
116
+ x1 = x;
117
+ y2 = y1;
118
+ y1 = y;
119
+ samples[index] = y;
120
+ }
121
+ }
122
+
123
+ /** The 2-8 kHz band of the signal: two high-pass and two low-pass sections (fourth order on each side). */
124
+ function getBand(mono: Float32Array, sampleRate: number): Float64Array {
125
+ const band = Float64Array.from(mono);
126
+ runBiquad(band, "highpass", BAND_LOW_HZ, sampleRate);
127
+ runBiquad(band, "highpass", BAND_LOW_HZ, sampleRate);
128
+ runBiquad(band, "lowpass", BAND_HIGH_HZ, sampleRate);
129
+ runBiquad(band, "lowpass", BAND_HIGH_HZ, sampleRate);
130
+ return band;
131
+ }
132
+
133
+ /** Each frame's mean square in dB, frames of `frameLength` samples every `hop` samples. */
134
+ function getFrameEnergies(band: Float64Array, frameLength: number, hop: number): Float64Array {
135
+ const sums = new Float64Array(band.length + 1);
136
+ for (let index = 0; index < band.length; index++) {
137
+ const value = band[index] ?? 0;
138
+ sums[index + 1] = (sums[index] ?? 0) + value * value;
139
+ }
140
+ const count = band.length < frameLength ? 0 : Math.floor((band.length - frameLength) / hop) + 1;
141
+ const energies = new Float64Array(count);
142
+ for (let frame = 0; frame < count; frame++) {
143
+ const from = frame * hop;
144
+ const power = ((sums[from + frameLength] ?? 0) - (sums[from] ?? 0)) / frameLength;
145
+ energies[frame] = 10 * Math.log10(Math.max(power, 1e-20));
146
+ }
147
+ return energies;
148
+ }
149
+
150
+ /** The index of the first entry of the sorted list not below `value`. */
151
+ function findInsertIndex(sorted: number[], value: number): number {
152
+ let low = 0;
153
+ let high = sorted.length;
154
+ while (low < high) {
155
+ const middle = (low + high) >> 1;
156
+ if ((sorted[middle] ?? 0) < value) low = middle + 1;
157
+ else high = middle;
158
+ }
159
+ return low;
160
+ }
161
+
162
+ /** The running median of the values over `reach` entries on each side (fewer at the ends). */
163
+ export function getRunningMedian(values: Float64Array, reach: number): Float64Array {
164
+ const medians = new Float64Array(values.length);
165
+ const window: number[] = [];
166
+ let from = 0;
167
+ let to = 0;
168
+ for (let index = 0; index < values.length; index++) {
169
+ const wantedTo = Math.min(values.length, index + reach + 1);
170
+ for (; to < wantedTo; to++) {
171
+ const value = values[to] ?? 0;
172
+ window.splice(findInsertIndex(window, value), 0, value);
173
+ }
174
+ const wantedFrom = Math.max(0, index - reach);
175
+ for (; from < wantedFrom; from++) window.splice(findInsertIndex(window, values[from] ?? 0), 1);
176
+ const middle = window.length >> 1;
177
+ medians[index] = window.length % 2 === 1 ? (window[middle] ?? 0) : ((window[middle - 1] ?? 0) + (window[middle] ?? 0)) / 2;
178
+ }
179
+ return medians;
180
+ }
181
+
182
+ /** True when the span overlaps a word's own span. */
183
+ const isInsideWord = (span: TimeSpan, words: readonly TimeSpan[]): boolean => words.some((word) => span.start < word.end && span.end > word.start);
184
+
185
+ /**
186
+ * The impulses of a mono signal: runs of 2 ms frames (1 ms hop) whose 2-8 kHz energy jumps more than the level's
187
+ * `jumpDb` above the band's running median (±100 ms), shorter than `DECLICK_MAX_IMPULSE`, overlapping no word's own
188
+ * span. Times in seconds, widened by a millisecond on each side.
189
+ */
190
+ export function findImpulses(mono: Float32Array, sampleRate: number, words: readonly TimeSpan[], level: ActiveDeclickLevel): TimeSpan[] {
191
+ const { jumpDb } = DECLICK_SETTINGS[level];
192
+ const frameLength = Math.max(1, Math.round(FRAME_SECONDS * sampleRate));
193
+ const hop = Math.max(1, Math.round(HOP_SECONDS * sampleRate));
194
+ const energies = getFrameEnergies(getBand(mono, sampleRate), frameLength, hop);
195
+ const medians = getRunningMedian(energies, Math.round(MEDIAN_SECONDS / HOP_SECONDS));
196
+ const spoken = sortSpans(words);
197
+ const seconds = mono.length / sampleRate;
198
+ const impulses: TimeSpan[] = [];
199
+ let runStart = -1;
200
+ const closeRun = (lastFrame: number): void => {
201
+ const start = (runStart * hop) / sampleRate;
202
+ const end = (lastFrame * hop + frameLength) / sampleRate;
203
+ runStart = -1;
204
+ if (end - start >= DECLICK_MAX_IMPULSE) return;
205
+ const span = { start: Math.max(0, start - IMPULSE_MARGIN), end: Math.min(seconds, end + IMPULSE_MARGIN) };
206
+ if (!isInsideWord(span, spoken)) impulses.push(span);
207
+ };
208
+ for (let frame = 0; frame < energies.length; frame++) {
209
+ const energy = energies[frame] ?? 0;
210
+ const isJump = energy > BAND_FLOOR_DB && energy - (medians[frame] ?? 0) > jumpDb;
211
+ if (isJump && runStart === -1) runStart = frame;
212
+ else if (!isJump && runStart !== -1) closeRun(frame - 1);
213
+ }
214
+ if (runStart !== -1) closeRun(energies.length - 1);
215
+ return impulses;
216
+ }
217
+
218
+ /** Sets the gain to `value` from sample `from` up to (not including) `to`, multiplying what is there. */
219
+ function scaleRange(gain: Float32Array, from: number, to: number, value: (index: number) => number): void {
220
+ for (let index = Math.max(0, from); index < Math.min(gain.length, to); index++) gain[index] = (gain[index] ?? 1) * value(index);
221
+ }
222
+
223
+ /** A linear ramp from `fromValue` at sample `from` to `toValue` at sample `to`. */
224
+ const ramp = (from: number, to: number, fromValue: number, toValue: number) => (index: number) => (to <= from ? toValue : fromValue + ((toValue - fromValue) * (index - from)) / (to - from));
225
+
226
+ /**
227
+ * The stage's gain per sample: the gate (1 inside the windows, a linear fade outside them, the gate's level beyond)
228
+ * times a dip for every impulse (its crossfades clamped to the neighbouring words, so no word is touched). With no
229
+ * words, no gate.
230
+ */
231
+ export function getDeclickGain(options: { length: number; sampleRate: number; words: readonly TimeSpan[]; impulses: readonly TimeSpan[]; level: ActiveDeclickLevel }): Float32Array {
232
+ const { length, sampleRate, level } = options;
233
+ const settings = DECLICK_SETTINGS[level];
234
+ const at = (seconds: number): number => Math.round(seconds * sampleRate);
235
+ const windows = getGateWindows(options.words);
236
+ const gate = windows.length === 0 ? 1 : fromDb(settings.gateDb);
237
+ const gain = new Float32Array(length).fill(1);
238
+ if (gate < 1) {
239
+ const fade = at(DECLICK_GATE_FADE);
240
+ const open = new Float32Array(length).fill(gate);
241
+ for (const window of windows) {
242
+ const start = at(window.start);
243
+ const end = at(window.end);
244
+ for (let index = Math.max(0, start - fade); index < Math.min(length, end + fade); index++) {
245
+ const value = index < start ? gate + ((1 - gate) * (index - (start - fade))) / fade : index >= end ? 1 - ((1 - gate) * (index - end)) / fade : 1;
246
+ open[index] = Math.max(open[index] ?? gate, value);
247
+ }
248
+ }
249
+ gain.set(open);
250
+ }
251
+ const spoken = sortSpans(options.words);
252
+ const dip = fromDb(DECLICK_IMPULSE_DB);
253
+ const crossfade = settings.crossfadeSeconds;
254
+ for (const impulse of options.impulses) {
255
+ const before = spoken.filter((word) => word.end <= impulse.start).reduce((latest, word) => Math.max(latest, word.end), 0);
256
+ const after = spoken.filter((word) => word.start >= impulse.end).reduce((earliest, word) => Math.min(earliest, word.start), Infinity);
257
+ const start = at(impulse.start);
258
+ const end = at(impulse.end);
259
+ const fadeIn = at(Math.max(before, impulse.start - crossfade));
260
+ const fadeOut = at(Math.min(after, impulse.end + crossfade));
261
+ scaleRange(gain, fadeIn, start, ramp(fadeIn, start, 1, dip));
262
+ scaleRange(gain, start, end, () => dip);
263
+ scaleRange(gain, end, fadeOut, ramp(end, fadeOut, dip, 1));
264
+ }
265
+ return gain;
266
+ }
267
+
268
+ /** The stage over interleaved samples: the impulses found on the channels' average, then the gain on every channel. */
269
+ export function declickSamples(options: { samples: Float32Array; channels: number; sampleRate: number; words: readonly TimeSpan[]; level: ActiveDeclickLevel }): {
270
+ samples: Float32Array;
271
+ impulses: TimeSpan[];
272
+ report: DeclickReport;
273
+ } {
274
+ const { samples, channels, sampleRate, words, level } = options;
275
+ const length = Math.floor(samples.length / channels);
276
+ const mono = new Float32Array(length);
277
+ for (let index = 0; index < length; index++) {
278
+ let sum = 0;
279
+ for (let channel = 0; channel < channels; channel++) sum += samples[index * channels + channel] ?? 0;
280
+ mono[index] = sum / channels;
281
+ }
282
+ const impulses = findImpulses(mono, sampleRate, words, level);
283
+ const gain = getDeclickGain({ length, sampleRate, words, impulses, level });
284
+ const output = new Float32Array(samples.length);
285
+ for (let index = 0; index < samples.length; index++) output[index] = (samples[index] ?? 0) * (gain[Math.floor(index / channels)] ?? 1);
286
+ const hasGate = getGateWindows(words).length > 0;
287
+ return { samples: output, impulses, report: { level, impulses: impulses.length, gateDb: hasGate ? DECLICK_SETTINGS[level].gateDb : null } };
288
+ }
289
+
290
+ const CHANNELS = 2;
291
+ const PCM_FORMAT = ["-f", "f32le", "-ar", String(FOOTAGE_SAMPLE_RATE), "-ac", String(CHANNELS)];
292
+
293
+ /** The cut speech decoded to raw 32-bit float PCM at the film's rate, stereo. */
294
+ export function buildDeclickDecodeArgs(input: string, output: string): string[] {
295
+ return ["-v", "error", "-y", "-i", input, "-vn", "-c:a", "pcm_f32le", ...PCM_FORMAT, output];
296
+ }
297
+
298
+ /** The declicked PCM encoded: WAV (float) when a loudness pass follows, else mp3 as the speech track is. */
299
+ export function buildDeclickEncodeArgs(input: string, output: string): string[] {
300
+ const codec = extname(output).toLowerCase() === ".wav" ? ["-c:a", "pcm_f32le"] : ["-c:a", "libmp3lame", "-q:a", "2"];
301
+ return ["-v", "error", "-y", ...PCM_FORMAT, "-i", input, ...codec, output];
302
+ }
303
+
304
+ /** Raw little-endian float32 bytes as samples (copied, so the view is aligned). */
305
+ function readFloats(file: string): Float32Array {
306
+ const bytes = readFileSync(file);
307
+ const samples = new Float32Array(Math.floor(bytes.length / 4));
308
+ for (let index = 0; index < samples.length; index++) samples[index] = bytes.readFloatLE(index * 4);
309
+ return samples;
310
+ }
311
+
312
+ function writeFloats(file: string, samples: Float32Array): void {
313
+ const bytes = Buffer.alloc(samples.length * 4);
314
+ for (let index = 0; index < samples.length; index++) bytes.writeFloatLE(samples[index] ?? 0, index * 4);
315
+ writeFileSync(file, bytes);
316
+ }
317
+
318
+ export interface DeclickSpeechOptions {
319
+ /** The cut speech track. */
320
+ input: string;
321
+ /** The declicked track: `.wav` (float) or `.mp3`. */
322
+ output: string;
323
+ /** The words in film time (the speech track's time). */
324
+ words: readonly TimedWord[];
325
+ level: ActiveDeclickLevel;
326
+ /** The track's seconds, for the progress: decoding is its first half, encoding its second. */
327
+ seconds: number;
328
+ signal?: AbortSignal;
329
+ onSeconds?: (done: number) => void;
330
+ }
331
+
332
+ /** Runs the stage on a speech track file: decode, find and lower, encode. The raw PCM next to the output is removed. */
333
+ export async function declickSpeech(options: DeclickSpeechOptions): Promise<DeclickReport> {
334
+ const { input, output, seconds } = options;
335
+ const aborting = options.signal === undefined ? {} : { signal: options.signal };
336
+ const raw = `${output}.f32`;
337
+ const report = (done: number): void => options.onSeconds?.(Math.min(seconds, Math.max(0, done)));
338
+ try {
339
+ await runFfmpeg("The speech declick's decode", buildDeclickDecodeArgs(input, raw), { ...aborting, output: raw, onSeconds: (done) => report(done / 2) });
340
+ const declicked = declickSamples({ samples: readFloats(raw), channels: CHANNELS, sampleRate: FOOTAGE_SAMPLE_RATE, words: options.words, level: options.level });
341
+ writeFloats(raw, declicked.samples);
342
+ await runFfmpeg("The speech declick's encode", buildDeclickEncodeArgs(raw, output), { ...aborting, output, onSeconds: (done) => report(seconds / 2 + done / 2) });
343
+ return declicked.report;
344
+ } finally {
345
+ rmSync(raw, { force: true });
346
+ }
347
+ }
348
+
349
+ /** The stage's log line. */
350
+ export function describeDeclick(report: DeclickReport): string {
351
+ const gate = report.gateDb === null ? "no words, so no gate" : `the pauses lowered by ${String(-report.gateDb)} dB`;
352
+ return `declick (${report.level}): ${String(report.impulses)} impulse${report.impulses === 1 ? "" : "s"} removed, ${gate}.`;
353
+ }