@softure-ai/marketing-kit 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +55 -0
  2. package/README.md +64 -8
  3. package/dist/cli/footage.d.ts.map +1 -1
  4. package/dist/cli/footage.js +5 -2
  5. package/dist/cli/footage.js.map +1 -1
  6. package/dist/compose/compose.d.ts.map +1 -1
  7. package/dist/compose/compose.js +3 -3
  8. package/dist/compose/compose.js.map +1 -1
  9. package/dist/compose/timeline.d.ts +16 -0
  10. package/dist/compose/timeline.d.ts.map +1 -1
  11. package/dist/compose/timeline.js +17 -1
  12. package/dist/compose/timeline.js.map +1 -1
  13. package/dist/config/config.d.ts.map +1 -1
  14. package/dist/config/config.js +18 -2
  15. package/dist/config/config.js.map +1 -1
  16. package/dist/config/schema.d.ts +16 -0
  17. package/dist/config/schema.d.ts.map +1 -1
  18. package/dist/config/schema.js +50 -3
  19. package/dist/config/schema.js.map +1 -1
  20. package/dist/footage/captions.d.ts +5 -1
  21. package/dist/footage/captions.d.ts.map +1 -1
  22. package/dist/footage/captions.js +6 -2
  23. package/dist/footage/captions.js.map +1 -1
  24. package/dist/footage/compose.d.ts +13 -2
  25. package/dist/footage/compose.d.ts.map +1 -1
  26. package/dist/footage/compose.js +75 -23
  27. package/dist/footage/compose.js.map +1 -1
  28. package/dist/footage/cuts.d.ts +44 -0
  29. package/dist/footage/cuts.d.ts.map +1 -0
  30. package/dist/footage/cuts.js +83 -0
  31. package/dist/footage/cuts.js.map +1 -0
  32. package/dist/footage/draft.d.ts +17 -3
  33. package/dist/footage/draft.d.ts.map +1 -1
  34. package/dist/footage/draft.js +28 -14
  35. package/dist/footage/draft.js.map +1 -1
  36. package/dist/footage/edl.d.ts +2 -1
  37. package/dist/footage/edl.d.ts.map +1 -1
  38. package/dist/footage/edl.js +5 -2
  39. package/dist/footage/edl.js.map +1 -1
  40. package/dist/footage/film.d.ts +42 -3
  41. package/dist/footage/film.d.ts.map +1 -1
  42. package/dist/footage/film.js +7 -1
  43. package/dist/footage/film.js.map +1 -1
  44. package/dist/footage/loudness.d.ts +20 -3
  45. package/dist/footage/loudness.d.ts.map +1 -1
  46. package/dist/footage/loudness.js +36 -18
  47. package/dist/footage/loudness.js.map +1 -1
  48. package/dist/footage/music.d.ts +3 -0
  49. package/dist/footage/music.d.ts.map +1 -1
  50. package/dist/footage/music.js +5 -0
  51. package/dist/footage/music.js.map +1 -1
  52. package/dist/footage/normalize.d.ts +9 -0
  53. package/dist/footage/normalize.d.ts.map +1 -1
  54. package/dist/footage/normalize.js +8 -4
  55. package/dist/footage/normalize.js.map +1 -1
  56. package/dist/footage/picture.d.ts +42 -0
  57. package/dist/footage/picture.d.ts.map +1 -0
  58. package/dist/footage/picture.js +141 -0
  59. package/dist/footage/picture.js.map +1 -0
  60. package/dist/footage/pipeline.d.ts +46 -1
  61. package/dist/footage/pipeline.d.ts.map +1 -1
  62. package/dist/footage/pipeline.js +128 -19
  63. package/dist/footage/pipeline.js.map +1 -1
  64. package/dist/footage/publish.d.ts +4 -2
  65. package/dist/footage/publish.d.ts.map +1 -1
  66. package/dist/footage/publish.js +8 -5
  67. package/dist/footage/publish.js.map +1 -1
  68. package/dist/footage/render.d.ts +17 -2
  69. package/dist/footage/render.d.ts.map +1 -1
  70. package/dist/footage/render.js +62 -13
  71. package/dist/footage/render.js.map +1 -1
  72. package/dist/index.d.ts +8 -6
  73. package/dist/index.d.ts.map +1 -1
  74. package/dist/index.js +8 -6
  75. package/dist/index.js.map +1 -1
  76. package/docker/footage-worker.Dockerfile +1 -0
  77. package/package.json +1 -1
  78. package/schema/marketing.schema.json +105 -4
  79. package/src/cli/footage.ts +4 -2
  80. package/src/compose/compose.ts +13 -3
  81. package/src/compose/timeline.ts +27 -1
  82. package/src/config/config.ts +18 -2
  83. package/src/config/schema.ts +52 -3
  84. package/src/footage/captions.ts +6 -2
  85. package/src/footage/compose.ts +86 -23
  86. package/src/footage/cuts.ts +104 -0
  87. package/src/footage/draft.ts +43 -14
  88. package/src/footage/edl.ts +5 -2
  89. package/src/footage/film.ts +47 -4
  90. package/src/footage/loudness.ts +38 -16
  91. package/src/footage/music.ts +7 -0
  92. package/src/footage/normalize.ts +8 -4
  93. package/src/footage/picture.ts +179 -0
  94. package/src/footage/pipeline.ts +166 -22
  95. package/src/footage/publish.ts +8 -5
  96. package/src/footage/render.ts +72 -13
  97. package/src/index.ts +36 -2
@@ -2,11 +2,11 @@ import type { BrandColors } from "../config/colors.js";
2
2
  import type { SfxEvent } from "../config/schema.js";
3
3
  import type { ComposeFont } from "../compose/compose.js";
4
4
  import { escapeHtml, fontFaces, fontStack } from "../compose/html.js";
5
- import { captionChunks, type CaptionLayout } from "../compose/timeline.js";
5
+ import { CAPTION_LINE_HEIGHT, captionChunks, getCaptionPillCss, getCaptionPositionCss, type CaptionLayout } from "../compose/timeline.js";
6
6
  import type { TimedWord } from "../voice/voiceover.js";
7
7
  import { getCaptionSpans, phraseChunks, type CaptionSpan } from "./captions.js";
8
8
  import { getEdlDuration, getSegmentStarts, type Edl } from "./edl.js";
9
- import { DEFAULT_EDIT, getFootageEdit, type FootageFilm } from "./film.js";
9
+ import { CAPTION_MIN_FONT_SCALE, DEFAULT_EDIT, getFootageEdit, type FootageEdit, type FootageFilm } from "./film.js";
10
10
  import { FOOTAGE_FPS } from "./normalize.js";
11
11
 
12
12
  const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
@@ -14,7 +14,7 @@ const r3 = (value: number): number => Math.round(value * 1000) / 1000 || 0;
14
14
  /**
15
15
  * The HyperFrames composition of a footage film (the same 0.8.85 contract as `composeFilm`): one `<video>` clip per
16
16
  * EDL segment over the normalized clip with its media start, the transitions (`fade` cross-fades from the segment
17
- * before over `fadeFrames`, `zoom` punches in by `zoomPercent` on alternate jump cuts, `whoosh` plays the whoosh
17
+ * before over `fadeFrames`, `zoom` punches in by `zoomPercent` on alternate jump cuts, eased over `zoomEaseFrames`, `whoosh` plays the whoosh
18
18
  * sound), a slow zoom on `kenburns` segments, the captions (plain phrases, or the word-highlight pill), the end card
19
19
  * over the cut's last frame (its whoosh only with `isEndCardWhoosh`), the cut speech track and the ducked music.
20
20
  */
@@ -28,8 +28,14 @@ const HAND_KEN_BURNS_PERCENT = 4;
28
28
  export const END_CARD_SECONDS = 3;
29
29
 
30
30
  export interface FootageComposeAssets {
31
- /** The normalized clips by id. */
31
+ /** The normalized clips by id (read when there is no `picture`). */
32
32
  clips: Record<string, string>;
33
+ /**
34
+ * The cut's picture in film time (`planFootagePicture`, cross-fades drawn in): the composition then plays it as one
35
+ * `<video>` and zooms that one element, instead of one `<video>` per segment. One video keeps hyperframes' frame
36
+ * extraction to one ffmpeg and its Chrome renderer lean (issue #1030). Absent: one `<video>` per segment.
37
+ */
38
+ picture?: string;
33
39
  /** The cut speech track (as long as the cut). */
34
40
  speech: string;
35
41
  /** The ducked music track, or null for none. */
@@ -71,6 +77,18 @@ export function getSegmentScales(edl: Edl, zoomPercent = DEFAULT_EDIT.zoomPercen
71
77
  return scales;
72
78
  }
73
79
 
80
+ /**
81
+ * The frames each segment's zoom eases over (`zoomEaseFrames`, at most the segment's own frames): a `zoom` cut after
82
+ * the first segment that changes the scale and is not `kenburns` (its slow zoom owns the scale); 0 for every other.
83
+ */
84
+ export function getSegmentZoomEases(edl: Edl, edit: Pick<FootageEdit, "zoomPercent" | "zoomEaseFrames">): number[] {
85
+ const scales = getSegmentScales(edl, edit.zoomPercent);
86
+ return edl.segments.map((segment, index) => {
87
+ const isEased = edit.zoomEaseFrames > 0 && index > 0 && segment.transition === "zoom" && segment.motion !== "kenburns" && scales[index - 1] !== scales[index];
88
+ return isEased ? Math.max(1, Math.min(edit.zoomEaseFrames, Math.floor((segment.end - segment.start) * FOOTAGE_FPS))) : 0;
89
+ });
90
+ }
91
+
74
92
  export function composeFootage(input: FootageComposeInput): string {
75
93
  const { film, edl, words, frame, colors: c, fonts, assets, caption: captionBox } = input;
76
94
  const starts = getSegmentStarts(edl);
@@ -78,31 +96,46 @@ export function composeFootage(input: FootageComposeInput): string {
78
96
  const end = getFootageFilmSeconds(film, edl);
79
97
  const edit = getFootageEdit(film);
80
98
  const scales = getSegmentScales(edl, edit.zoomPercent);
99
+ const eases = getSegmentZoomEases(edl, edit);
81
100
  const fadeSeconds = r3(edit.fadeFrames / FOOTAGE_FPS);
82
101
  const kenBurnsScale = 1 + (edit.kenBurnsPercent > 0 ? edit.kenBurnsPercent : HAND_KEN_BURNS_PERCENT) / 100;
83
102
 
84
- const segments = edl.segments.map((segment, index) => {
85
- const src = assets.clips[segment.clip];
86
- if (src === undefined) throw new Error(`The EDL names clip "${segment.clip}", which the film has no file for.`);
87
- const next = edl.segments[index + 1];
88
- const tail = next?.transition === "fade" ? fadeSeconds : 0;
89
- const start = starts[index] as number;
90
- const duration = r3(segment.end - segment.start + tail);
91
- const scale = scales[index] as number;
92
- const style = scale === 1 ? "" : ` style="transform:scale(${scale})"`;
93
- return `<video id="seg${index}" class="clip shot" src="${escapeHtml(src)}" data-start="${start}" data-media-start="${segment.start}" data-duration="${duration}"${style} muted playsinline></video>`;
94
- });
103
+ const picture = assets.picture;
104
+ const segments =
105
+ picture === undefined
106
+ ? edl.segments.map((segment, index) => {
107
+ const src = assets.clips[segment.clip];
108
+ if (src === undefined) throw new Error(`The EDL names clip "${segment.clip}", which the film has no file for.`);
109
+ const next = edl.segments[index + 1];
110
+ const tail = next?.transition === "fade" ? fadeSeconds : 0;
111
+ const start = starts[index] as number;
112
+ const duration = r3(segment.end - segment.start + tail);
113
+ const scale = scales[index] as number;
114
+ const style = scale === 1 ? "" : ` style="transform:scale(${scale})"`;
115
+ return `<video id="seg${index}" class="clip shot" src="${escapeHtml(src)}" data-start="${start}" data-media-start="${segment.start}" data-duration="${duration}"${style} muted playsinline></video>`;
116
+ })
117
+ : [`<video id="picture" class="clip shot" src="${escapeHtml(picture)}" data-start="0" data-media-start="0" data-duration="${cutEnd}" muted playsinline></video>`];
95
118
 
96
119
  const tweens: string[] = [];
97
120
  edl.segments.forEach((segment, index) => {
98
121
  const start = starts[index] as number;
99
- if (segment.transition === "fade" && index > 0) {
100
- tweens.push(`tl.fromTo("#seg${index}", { opacity: 0 }, { opacity: 1, duration: ${fadeSeconds}, ease: "none", immediateRender: false }, ${start});`);
122
+ const scale = scales[index] as number;
123
+ const target = picture === undefined ? `#seg${index}` : "#picture";
124
+ // One video carries every segment: each one sets the zoom it starts at (a fade is already in the picture).
125
+ if (picture !== undefined) tweens.push(`tl.set("#picture", { scale: ${scale} }, ${start});`);
126
+ if (picture === undefined && segment.transition === "fade" && index > 0) {
127
+ tweens.push(`tl.fromTo("${target}", { opacity: 0 }, { opacity: 1, duration: ${fadeSeconds}, ease: "none", immediateRender: false }, ${start});`);
128
+ }
129
+ const ease = eases[index] as number;
130
+ if (ease > 0) {
131
+ // Eases from the scale on screen before the cut, so the punch-in (or back out) reads as one move of the camera.
132
+ tweens.push(
133
+ `tl.fromTo("${target}", { scale: ${scales[index - 1] as number} }, { scale: ${scales[index] as number}, duration: ${r3(ease / FOOTAGE_FPS)}, ease: "power1.out", immediateRender: false }, ${start});`,
134
+ );
101
135
  }
102
136
  if (segment.motion === "kenburns") {
103
- const from = scales[index] as number;
104
137
  tweens.push(
105
- `tl.fromTo("#seg${index}", { scale: ${from} }, { scale: ${r3(from * kenBurnsScale)}, duration: ${r3(segment.end - segment.start)}, ease: "none", immediateRender: false }, ${start});`,
138
+ `tl.fromTo("${target}", { scale: ${scale} }, { scale: ${r3(scale * kenBurnsScale)}, duration: ${r3(segment.end - segment.start)}, ease: "none", immediateRender: false }, ${start});`,
106
139
  );
107
140
  }
108
141
  });
@@ -121,6 +154,8 @@ export function composeFootage(input: FootageComposeInput): string {
121
154
  })
122
155
  .join("\n ");
123
156
 
157
+ const captionFit = film.captions && film.captionMaxLines !== undefined ? getCaptionFitScript(film.captionMaxLines, Math.round(captionBox.fontSize * CAPTION_MIN_FONT_SCALE)) : "";
158
+
124
159
  // --- Sound ---
125
160
  const sfx: { src: string; t: number; volume: number; dur: number }[] = [];
126
161
  const play = (event: SfxEvent, t: number, volume: number, dur: number) => {
@@ -182,10 +217,10 @@ html,body{width:${frame.width}px;height:${frame.height}px;overflow:hidden;backgr
182
217
  #root{position:relative;width:${frame.width}px;height:${frame.height}px;overflow:hidden;background:#000;font-family:${bodyFont};color:${c.foreground}}
183
218
  .shot{position:absolute;inset:0;width:100%;height:100%;object-fit:cover;transform-origin:50% 42%}
184
219
  .shade{position:absolute;inset:0;background:${c.background};opacity:0}
185
- .caption{position:absolute;left:${captionBox.left}px;right:${captionBox.right}px;top:${captionBox.top}px;display:flex;justify-content:center}
186
- .pill{font-weight:650;font-size:${captionBox.fontSize}px;line-height:1.18;letter-spacing:-.01em;text-align:center;padding:16px 28px;border-radius:22px;background:${c.captionBackground};box-shadow:0 14px 44px rgba(0,0,0,.5);color:${c.captionText}}
220
+ .caption{position:absolute;left:${captionBox.left}px;right:${captionBox.right}px;${getCaptionPositionCss(captionBox)};display:flex;justify-content:center}
221
+ .pill{${getCaptionPillCss(captionBox.fontSize)};border-radius:22px;background:${c.captionBackground};box-shadow:0 14px 44px rgba(0,0,0,.5);color:${c.captionText}}
187
222
  .pill span{display:inline-block}
188
- .phrase{font-weight:650;font-size:${captionBox.fontSize}px;line-height:1.18;letter-spacing:-.01em;text-align:center;color:#fff;text-shadow:0 2px 6px rgba(0,0,0,.55),0 0 18px rgba(0,0,0,.35)}
223
+ .phrase{font-weight:650;font-size:${captionBox.fontSize}px;line-height:${CAPTION_LINE_HEIGHT};letter-spacing:-.01em;text-align:center;color:#fff;text-shadow:0 2px 6px rgba(0,0,0,.55),0 0 18px rgba(0,0,0,.35)}
189
224
  .endcard{position:absolute;inset:0;display:flex;flex-direction:column;justify-content:center;align-items:center;padding:0 80px;text-align:center}
190
225
  .brand{display:flex;justify-content:center;align-items:center;gap:18px;font-family:${headingFont};font-size:46px}
191
226
  .logo{height:60px;width:auto}
@@ -201,7 +236,7 @@ html,body{width:${frame.width}px;height:${frame.height}px;overflow:hidden;backgr
201
236
  ${captionHtml}
202
237
  ${audio}
203
238
  </div>
204
- <script>
239
+ ${captionFit}<script>
205
240
  const tl = gsap.timeline({ paused: true });
206
241
  ${tweens.join("\n")}
207
242
  const captions = ${JSON.stringify(isHighlight ? caption.map((chunk) => chunk.words.map((w) => w.start)) : [])};
@@ -218,3 +253,31 @@ window.__timelines["${COMPOSITION_ID}"] = tl;
218
253
  </html>
219
254
  `;
220
255
  }
256
+
257
+ /**
258
+ * A page script that shrinks each caption's font in 1 px steps until its text takes at most `maxLines` lines, or the
259
+ * font reaches `minFontSize`. Only the browser knows how the font wraps, so it measures there: once at load and again
260
+ * when the fonts are ready (hyperframes waits for them before the first frame). Hidden clips keep their layout.
261
+ */
262
+ function getCaptionFitScript(maxLines: number, minFontSize: number): string {
263
+ return `<script>
264
+ (() => {
265
+ const captionMaxLines = ${maxLines}, captionMinFontSize = ${minFontSize};
266
+ const countLines = (el) => {
267
+ const style = getComputedStyle(el);
268
+ return Math.round((el.clientHeight - parseFloat(style.paddingTop) - parseFloat(style.paddingBottom)) / parseFloat(style.lineHeight));
269
+ };
270
+ const fit = () => document.querySelectorAll(".caption > p").forEach((el) => {
271
+ el.style.fontSize = "";
272
+ let size = parseFloat(getComputedStyle(el).fontSize);
273
+ while (countLines(el) > captionMaxLines && size > captionMinFontSize) {
274
+ size -= 1;
275
+ el.style.fontSize = size + "px";
276
+ }
277
+ });
278
+ fit();
279
+ if (document.fonts) document.fonts.ready.then(fit);
280
+ })();
281
+ </script>
282
+ `;
283
+ }
@@ -0,0 +1,104 @@
1
+ import { mkdirSync, readFileSync, rmSync } from "node:fs";
2
+ import { join } from "node:path";
3
+
4
+ import type { Edl } from "./edl.js";
5
+ import { DEFAULT_AUTO_THRESHOLD } from "./film.js";
6
+ import { FOOTAGE_FPS } from "./normalize.js";
7
+ import { runFfmpeg } from "./process.js";
8
+
9
+ /**
10
+ * Transitions chosen from the frames around each jump cut (`transitions.jump: "auto"`): the last frame before a cut
11
+ * and the first after it, both from the normalized clips, are shrunk to a small gray thumbnail and compared. A small
12
+ * difference (the same framing, a removed filler) gets an eased punch-in (`zoom`), a large one (the camera moved) a
13
+ * short dissolve (`fade`). Deterministic and local: ffmpeg only, no provider.
14
+ */
15
+
16
+ /** The thumbnail both frames are compared at: small enough that handheld shake and noise average out. */
17
+ export const CUT_THUMBNAIL = { width: 48, height: 48 } as const;
18
+
19
+ /** What the frames around one cut measured and the transition it got. */
20
+ export interface CutMeasure {
21
+ /** The id of the segment the cut starts. */
22
+ segment: string;
23
+ /** The mean absolute difference of the two thumbnails' gray levels, 0 (the same) to 1 (black against white). */
24
+ difference: number;
25
+ transition: "zoom" | "fade";
26
+ }
27
+
28
+ const r4 = (value: number): number => Math.round(value * 10000) / 10000 || 0;
29
+
30
+ /** The mean absolute difference of two gray frames of the same size, 0-1. */
31
+ export function measureFrameDifference(before: Uint8Array, after: Uint8Array): number {
32
+ if (before.length === 0 || before.length !== after.length) {
33
+ throw new Error(`Comparing the frames around a cut needs two frames of one size, got ${before.length} and ${after.length} bytes.`);
34
+ }
35
+ let sum = 0;
36
+ for (let index = 0; index < before.length; index++) sum += Math.abs((before[index] as number) - (after[index] as number));
37
+ return r4(sum / before.length / 255);
38
+ }
39
+
40
+ /** `fade` from `threshold` up (the picture moved), `zoom` below it (the same shot). */
41
+ export function chooseCutTransition(difference: number, threshold: number = DEFAULT_AUTO_THRESHOLD): "zoom" | "fade" {
42
+ return difference >= threshold ? "fade" : "zoom";
43
+ }
44
+
45
+ /** One frame of a clip at `seconds` as a raw gray thumbnail (`CUT_THUMBNAIL`). */
46
+ export function buildCutFrameArgs(file: string, seconds: number, output: string): string[] {
47
+ const { width, height } = CUT_THUMBNAIL;
48
+ return ["-v", "error", "-y", "-ss", String(seconds), "-i", file, "-frames:v", "1", "-vf", `scale=${width}:${height}:flags=area,format=gray`, "-f", "rawvideo", output];
49
+ }
50
+
51
+ /** The indexes of the jump cuts the auto choice sets: a plain cut inside one clip, after the first segment. */
52
+ export function findAutoCuts(edl: Edl): number[] {
53
+ return edl.segments.flatMap((segment, index) => {
54
+ const previous = edl.segments[index - 1];
55
+ return previous !== undefined && previous.clip === segment.clip && segment.transition === "cut" ? [index] : [];
56
+ });
57
+ }
58
+
59
+ export interface ChooseCutTransitionsOptions {
60
+ /** The difference from which a cut fades; default `DEFAULT_AUTO_THRESHOLD`. */
61
+ threshold?: number;
62
+ /** A folder for the thumbnails; created, and removed when done. */
63
+ workDir: string;
64
+ signal?: AbortSignal;
65
+ }
66
+
67
+ /**
68
+ * Measures every plain jump cut of `edl` (`findAutoCuts`) on the normalized clips and gives it `zoom` or `fade`. Cuts
69
+ * between clips and every other transition stay as they are. Answers the new EDL and what each cut measured.
70
+ */
71
+ export async function chooseCutTransitions(edl: Edl, clips: Record<string, string>, options: ChooseCutTransitionsOptions): Promise<{ edl: Edl; cuts: CutMeasure[] }> {
72
+ const threshold = options.threshold ?? DEFAULT_AUTO_THRESHOLD;
73
+ if (!(threshold >= 0 && threshold <= 1)) throw new Error(`transitions.autoThreshold must be between 0 and 1, got ${String(threshold)}.`);
74
+ const indexes = findAutoCuts(edl);
75
+ if (indexes.length === 0) return { edl, cuts: [] };
76
+ const signal = options.signal === undefined ? {} : { signal: options.signal };
77
+ const getClip = (id: string): string => {
78
+ const file = clips[id];
79
+ if (file === undefined) throw new Error(`No file for clip "${id}" of the EDL.`);
80
+ return file;
81
+ };
82
+ mkdirSync(options.workDir, { recursive: true });
83
+ const segments = [...edl.segments];
84
+ const cuts: CutMeasure[] = [];
85
+ try {
86
+ for (const index of indexes) {
87
+ const segment = segments[index] as Edl["segments"][number];
88
+ const previous = segments[index - 1] as Edl["segments"][number];
89
+ const beforeFile = join(options.workDir, `${segment.id}-before.gray`);
90
+ const afterFile = join(options.workDir, `${segment.id}-after.gray`);
91
+ // The last frame the cut shows before it, and the first after it.
92
+ const beforeAt = Math.round(Math.max(previous.start, previous.end - 1 / FOOTAGE_FPS) * 1000) / 1000;
93
+ await runFfmpeg(`The frame before cut ${segment.id}`, buildCutFrameArgs(getClip(previous.clip), beforeAt, beforeFile), { ...signal, output: beforeFile });
94
+ await runFfmpeg(`The frame after cut ${segment.id}`, buildCutFrameArgs(getClip(segment.clip), segment.start, afterFile), { ...signal, output: afterFile });
95
+ const difference = measureFrameDifference(readFileSync(beforeFile), readFileSync(afterFile));
96
+ const transition = chooseCutTransition(difference, threshold);
97
+ segments[index] = { ...segment, transition };
98
+ cuts.push({ segment: segment.id, difference, transition });
99
+ }
100
+ } finally {
101
+ rmSync(options.workDir, { recursive: true, force: true });
102
+ }
103
+ return { edl: { ...edl, segments }, cuts };
104
+ }
@@ -6,17 +6,17 @@ import type { CaptionLayout, LayoutOverride } from "../compose/timeline.js";
6
6
  import { ok } from "../voice/provider.js";
7
7
  import type { TimedWord } from "../voice/voiceover.js";
8
8
  import { getCaptionSpans, getFootageCaptionLayout, phraseChunks } from "./captions.js";
9
- import { getSegmentScales } from "./compose.js";
9
+ import { getSegmentScales, getSegmentZoomEases } from "./compose.js";
10
10
  import { getEdlDuration, getSegmentStarts, remapWords, type Edl } from "./edl.js";
11
11
  import { getFootageEdit, type FootageFilm } from "./film.js";
12
- import { FOOTAGE_FPS, type Frame } from "./normalize.js";
12
+ import { FOOTAGE_FPS, getThreadArgs, type Frame } from "./normalize.js";
13
13
  import { getFrame, type FootageResult, type FootageRunOptions, type PreparedClip, type TranscribeOutcome } from "./pipeline.js";
14
14
  import { runFfmpeg } from "./process.js";
15
- import { buildSpeechArgs } from "./render.js";
15
+ import { buildSpeechArgs, getSpeechCutOptions } from "./render.js";
16
16
 
17
17
  /**
18
18
  * A fast draft of a footage film, ffmpeg only (no hyperframes, no browser): the cut at a small width with the film's
19
- * phrase captions burned in, its zoom cuts, cross-fades and Ken Burns b-roll drawn as the full render draws them, and
19
+ * phrase captions burned in, its zoom cuts (eased as in the render), cross-fades and Ken Burns b-roll drawn as the full render draws them, and
20
20
  * the speech track. No loudness pass, music, end card or cover. What an app shows the owner before the paid render.
21
21
  */
22
22
 
@@ -75,8 +75,8 @@ export interface FootageDraftCaptionsInput {
75
75
 
76
76
  /**
77
77
  * The draft's captions as an ASS subtitle file: the film's phrases (`phraseChunks`) shown for the render's spans
78
- * (`getCaptionSpans`), white bold text with a dark outline and shadow, centred in the caption box from its top. Null
79
- * when the film has no captions or nothing is said.
78
+ * (`getCaptionSpans`), white bold text with a dark outline and shadow, centred in the caption box from its top (or
79
+ * from its foot, with `bottom`). Null when the film has no captions or nothing is said.
80
80
  */
81
81
  export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): string | null {
82
82
  if (!input.film.captions) return null;
@@ -106,10 +106,11 @@ export function buildFootageDraftCaptions(input: FootageDraftCaptionsInput): str
106
106
  1,
107
107
  outline,
108
108
  shadow,
109
- 8,
109
+ // ASS alignment 2 hangs the text from MarginV above the bottom, 8 from MarginV below the top.
110
+ caption.bottom === undefined ? 8 : 2,
110
111
  Math.round(caption.left),
111
112
  Math.round(caption.right),
112
- Math.round(caption.top),
113
+ Math.round(caption.bottom ?? caption.top),
113
114
  1,
114
115
  ].join(",");
115
116
  const events = chunks.map((chunk, index) => {
@@ -152,6 +153,11 @@ export interface FootageDraftArgsInput {
152
153
  /** The folder libass loads the captions' font from, or null for the system's fonts. */
153
154
  fontsDir: string | null;
154
155
  output: string;
156
+ /**
157
+ * ffmpeg's threads for every segment's decoder, the filter graph and the encoder; by default ffmpeg's own choice,
158
+ * which follows the host's cores, not a container's CPU quota. A positive whole number.
159
+ */
160
+ threads?: number;
155
161
  }
156
162
 
157
163
  /** A segment's picture as the draft plays it: how long its stream runs and the fade it starts with. */
@@ -168,6 +174,10 @@ export interface DraftSegmentPlan {
168
174
  scale: number;
169
175
  /** The zoom it ends at: above `scale` for a `kenburns` segment, `scale` otherwise. */
170
176
  endScale: number;
177
+ /** The zoom on screen before an eased `zoom` cut, which the segment eases from to `scale`; `scale` otherwise. */
178
+ startScale: number;
179
+ /** The frames the zoom eases over (`getSegmentZoomEases`); 0: the segment starts at `scale`. */
180
+ easeFrames: number;
171
181
  }
172
182
 
173
183
  /**
@@ -179,6 +189,7 @@ export function planFootageDraft(film: Pick<FootageFilm, "edit">, edl: Edl): Dra
179
189
  const fadeSeconds = r3(edit.fadeFrames / FOOTAGE_FPS);
180
190
  const kenBurnsScale = 1 + (edit.kenBurnsPercent > 0 ? edit.kenBurnsPercent : HAND_KEN_BURNS_PERCENT) / 100;
181
191
  const scales = getSegmentScales(edl, edit.zoomPercent);
192
+ const eases = getSegmentZoomEases(edl, edit);
182
193
  const starts = getSegmentStarts(edl);
183
194
  const fades = edl.segments.map((segment, index) => (segment.transition === "fade" && index > 0 ? r3(Math.min(fadeSeconds, segment.end - segment.start)) : 0));
184
195
  return edl.segments.map((segment, index) => {
@@ -190,20 +201,28 @@ export function planFootageDraft(film: Pick<FootageFilm, "edit">, edl: Edl): Dra
190
201
  fade: fades[index] as number,
191
202
  scale,
192
203
  endScale: segment.motion === "kenburns" ? r4(scale * kenBurnsScale) : scale,
204
+ startScale: (eases[index] as number) > 0 ? (scales[index - 1] as number) : scale,
205
+ easeFrames: eases[index] as number,
193
206
  };
194
207
  });
195
208
  }
196
209
 
197
- /** The filters that zoom a segment's picture and scale it to the draft: a crop for a still zoom, `zoompan` for Ken Burns. */
210
+ /**
211
+ * The filters that zoom a segment's picture and scale it to the draft: a crop for a still zoom, `zoompan` for Ken Burns
212
+ * and for an eased zoom cut (the render's `power1.out`: `1 - (1 - t)^2`).
213
+ */
198
214
  function buildPictureFilters(plan: DraftSegmentPlan, draft: Frame): string[] {
199
215
  const size = `${draft.width}:${draft.height}`;
216
+ const zoompan = (zoom: string): string =>
217
+ `zoompan=z='${zoom}':x='(iw-iw/zoom)*${ZOOM_ORIGIN.x}':y='(ih-ih/zoom)*${ZOOM_ORIGIN.y}':d=1:s=${draft.width}x${draft.height}:fps=${FOOTAGE_FPS}`;
218
+ if (plan.easeFrames > 0) {
219
+ return [zoompan(`${plan.startScale}+(${r4(plan.scale - plan.startScale)})*(1-pow(1-min(on/${plan.easeFrames},1),2))`)];
220
+ }
200
221
  if (plan.endScale !== plan.scale) {
201
222
  // The zoom runs over the segment's own length, as the render's tween does, and holds through a fade's tail.
202
223
  const frames = Math.max(1, Math.round(plan.length * FOOTAGE_FPS));
203
224
  const zoom = `${plan.scale}+${r4(plan.endScale - plan.scale)}*min(on/${frames},1)`;
204
- return [
205
- `zoompan=z='${zoom}':x='(iw-iw/zoom)*${ZOOM_ORIGIN.x}':y='(ih-ih/zoom)*${ZOOM_ORIGIN.y}':d=1:s=${draft.width}x${draft.height}:fps=${FOOTAGE_FPS}`,
206
- ];
225
+ return [zoompan(zoom)];
207
226
  }
208
227
  if (plan.scale !== 1) {
209
228
  return [`crop=iw/${plan.scale}:ih/${plan.scale}:(iw-iw/${plan.scale})*${ZOOM_ORIGIN.x}:(ih-ih/${plan.scale})*${ZOOM_ORIGIN.y}`, `scale=${size}`];
@@ -220,13 +239,15 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
220
239
  const { edl } = input;
221
240
  const draft = getFootageDraftFrame(input.frame, input.width);
222
241
  const plans = planFootageDraft(input.film, edl);
242
+ const threads = getThreadArgs(input.threads, "draft.threads");
223
243
  const inputs: string[] = [];
224
244
  const parts: string[] = [];
225
245
  edl.segments.forEach((segment, index) => {
226
246
  const clip = input.clips[segment.clip];
227
247
  if (clip === undefined) throw new Error(`No file for clip "${segment.clip}" of the EDL.`);
228
248
  const plan = plans[index] as DraftSegmentPlan;
229
- inputs.push("-ss", String(r3(segment.start)), "-t", String(plan.seconds), "-i", clip);
249
+ // Before an input, -threads sizes that input's decoder: one decoder runs per segment.
250
+ inputs.push(...threads.output, "-ss", String(r3(segment.start)), "-t", String(plan.seconds), "-i", clip);
230
251
  const filters = [
231
252
  "setpts=PTS-STARTPTS",
232
253
  ...buildPictureFilters(plan, draft),
@@ -264,6 +285,7 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
264
285
  "-v",
265
286
  "error",
266
287
  "-y",
288
+ ...threads.global,
267
289
  ...inputs,
268
290
  "-i",
269
291
  input.speech,
@@ -289,6 +311,7 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
289
311
  DRAFT_AUDIO_BITRATE,
290
312
  "-movflags",
291
313
  "+faststart",
314
+ ...threads.output,
292
315
  input.output,
293
316
  ];
294
317
  }
@@ -300,6 +323,11 @@ export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buil
300
323
  width?: number;
301
324
  /** A project's override of the format's caption box (`marketing.json` `layout`), as the render takes it. */
302
325
  layout?: LayoutOverride;
326
+ /**
327
+ * ffmpeg's threads for the draft's decoders, filter graph and encoder (`buildFootageDraftArgs`); by default ffmpeg's
328
+ * own choice, which follows the host's cores. Set it from a CPU-limited container's quota.
329
+ */
330
+ threads?: number;
303
331
  }
304
332
 
305
333
  export interface FootageDraft {
@@ -343,7 +371,7 @@ export async function renderFootageDraft(
343
371
  const output = resolve(options.output);
344
372
  mkdirSync(dirname(output), { recursive: true });
345
373
  report("speech", 1, 0);
346
- await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech), { ...aborting, output: speech, onSeconds: (done) => report("speech", 1, done) });
374
+ await runFfmpeg("The draft's speech track", buildSpeechArgs(edl, transcript.speech, speech, getSpeechCutOptions(getFootageEdit(film))), { ...aborting, output: speech, onSeconds: (done) => report("speech", 1, done) });
347
375
  report("film", 2, 0);
348
376
  // Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
349
377
  const args = buildFootageDraftArgs({
@@ -353,6 +381,7 @@ export async function renderFootageDraft(
353
381
  speech,
354
382
  frame,
355
383
  ...(options.width === undefined ? {} : { width: options.width }),
384
+ ...(options.threads === undefined ? {} : { threads: options.threads }),
356
385
  captions: captions === null ? null : "captions.ass",
357
386
  fontsDir: captions === null ? null : "fonts",
358
387
  output,
@@ -185,7 +185,8 @@ function claimId(base: string, used: Set<string>): string {
185
185
  * The cut from the transcripts. A talking clip keeps its runs of words: fillers are cut out, a pause longer than
186
186
  * `pauseSeconds` is cut, `padSeconds` stays around each cut, and with `keepSentences` only those sentences stay. A
187
187
  * talking clip without words, and every b-roll clip, is kept whole. The first segment starts with a cut; later ones
188
- * with `transitions.jump` inside one clip and `transitions.clip` between clips. Each segment's id names the sentence
188
+ * with `transitions.jump` inside one clip (a plain cut for `auto`, chosen later from the frames by
189
+ * `chooseCutTransitions`) and `transitions.clip` between clips. Each segment's id names the sentence
189
190
  * it starts with (`s4`), or the b-roll (`r1`) or wordless clip (`w1`) it shows.
190
191
  */
191
192
  export function buildEdl(filmId: string, clips: EdlClip[], options: CutOptions, transitions: FootageTransitions, edit: BuildEdlOptions = {}): Edl {
@@ -204,7 +205,9 @@ export function buildEdl(filmId: string, clips: EdlClip[], options: CutOptions,
204
205
  // A talking clip whose sentences were all dropped (or cut to nothing) leaves the film.
205
206
  for (const span of spans) {
206
207
  const previous = segments.at(-1);
207
- const transition = previous === undefined ? "cut" : previous.clip === clip.id ? transitions.jump : transitions.clip;
208
+ // An auto jump cut starts as a plain cut; `chooseCutTransitions` sets it from the frames.
209
+ const jump = transitions.jump === "auto" ? "cut" : transitions.jump;
210
+ const transition = previous === undefined ? "cut" : previous.clip === clip.id ? jump : transitions.clip;
208
211
  const base = span.sentence ?? (clip.role === "broll" ? `r${++broll}` : `w${++wordless}`);
209
212
  const motion = clip.role === "broll" && edit.isKenBurns === true ? "kenburns" : "none";
210
213
  segments.push({ id: claimId(base, used), clip: clip.id, start: r3(span.start), end: r3(span.end), transition, motion });
@@ -64,6 +64,10 @@ export interface FootageMusic {
64
64
  volume: number;
65
65
  /** The music's volume under speech (0-1). */
66
66
  duckedVolume: number;
67
+ /** Seconds the music takes to go down before a stretch of speech. Absent: 0.25. */
68
+ attack?: number;
69
+ /** Seconds the music takes to come back up after a stretch of speech. Absent: 0.5. */
70
+ release?: number;
67
71
  }
68
72
 
69
73
  /** How the transcript turns into the cut (`buildEdl`). */
@@ -78,14 +82,27 @@ export interface CutOptions {
78
82
  maxSeconds: number | null;
79
83
  }
80
84
 
85
+ /** A jump cut's default: a transition, or `auto` to choose `zoom` or `fade` from the frames around each cut. */
86
+ export const JUMP_TRANSITIONS = [...CUT_TRANSITIONS, "auto"] as const;
87
+
88
+ export type JumpTransition = (typeof JUMP_TRANSITIONS)[number];
89
+
81
90
  /** Default transitions by kind of cut; a segment of the EDL may name its own. */
82
91
  export interface FootageTransitions {
83
- /** A cut inside one talking clip (a removed filler or pause). */
84
- jump: CutTransition;
92
+ /**
93
+ * A cut inside one talking clip (a removed filler or pause). `auto`: when the cut is built, the frames on both sides
94
+ * are compared (`chooseCutTransitions`): a small difference gets a `zoom`, a large one (the camera moved) a `fade`.
95
+ */
96
+ jump: JumpTransition;
85
97
  /** A cut from one clip to another. */
86
98
  clip: CutTransition;
99
+ /** With `jump: "auto"`, the frame difference (0-1) from which a jump cut fades instead of zooming; default 0.08. */
100
+ autoThreshold?: number;
87
101
  }
88
102
 
103
+ /** The frame difference (0-1) from which an `auto` jump cut fades instead of zooming. */
104
+ export const DEFAULT_AUTO_THRESHOLD = 0.08;
105
+
89
106
  export interface FootageFilm {
90
107
  id: string;
91
108
  title: string;
@@ -109,12 +126,23 @@ export interface FootageFilm {
109
126
  captionStyle?: CaptionStyle;
110
127
  /** Places the 9:16 caption box inside a platform's safe zone (`CAPTION_SAFE_ZONES`); absent: the layout as it is. */
111
128
  captionPreset?: CaptionPreset;
129
+ /**
130
+ * The most lines a caption card may take: the render shrinks a longer card's font (to at most
131
+ * `CAPTION_MIN_FONT_SCALE` of the box's size) until it fits. Absent: no limit. The draft keeps its font size.
132
+ */
133
+ captionMaxLines?: number;
112
134
  /** The cut's look: zoom size, fade length, Ken Burns, the end card's whoosh. Absent fields take `DEFAULT_EDIT`. */
113
135
  edit?: Partial<FootageEdit>;
114
136
  /** The closing card after the last segment; null: the film ends with its last segment. */
115
137
  endCard: EndCard | null;
116
138
  /** The speech track's loudness, normalized before the music is mixed under it. Absent: −14 LUFS, −1 dBTP; null: left as filmed. */
117
139
  loudness?: LoudnessTarget | null;
140
+ /**
141
+ * The finished film's mix (speech, music, sound effects), measured after the render and normalized in two passes of
142
+ * `loudnorm` (one linear gain; the picture is copied, only the sound re-encoded), so the published file plays at the
143
+ * target. A silent mix is left as it is. Absent: −14 LUFS, −1 dBTP; null: the mix as rendered.
144
+ */
145
+ mixLoudness?: LoudnessTarget | null;
118
146
  /**
119
147
  * The cover JPG written next to the film. Absent or without `at`: the middle of the first talking segment; `at`:
120
148
  * that second of the film; null: no cover.
@@ -142,22 +170,37 @@ export const CAPTION_STYLES = ["phrase", "highlight"] as const;
142
170
 
143
171
  export type CaptionStyle = (typeof CAPTION_STYLES)[number];
144
172
 
173
+ /** The smallest a caption line limit shrinks a card's font to, as a share of the caption box's size. */
174
+ export const CAPTION_MIN_FONT_SCALE = 0.7;
175
+
145
176
  export const CAPTION_PRESETS = ["reels", "shorts", "tiktok", "all"] as const;
146
177
 
147
178
  export type CaptionPreset = (typeof CAPTION_PRESETS)[number];
148
179
 
149
180
  export interface FootageEdit {
150
- /** How far a `zoom` cut punches in, in percent of the frame (105-110). */
181
+ /** How far a `zoom` cut punches in, in percent of the frame (105-112). */
151
182
  zoomPercent: number;
183
+ /**
184
+ * Frames at 30 fps a `zoom` cut eases into its scale over, from the scale on screen before the cut (0-15); 0 steps
185
+ * straight to it. A `kenburns` segment keeps its own slow zoom and is not eased.
186
+ */
187
+ zoomEaseFrames: number;
152
188
  /** A `fade` cut's length in frames at 30 fps (1-30). */
153
189
  fadeFrames: number;
154
190
  /** How far b-roll zooms over its length (0-5 %); 0 keeps b-roll still and `buildEdl` marks none `kenburns`. */
155
191
  kenBurnsPercent: number;
156
192
  /** Play the whoosh (and pop) as the end card comes in. */
157
193
  isEndCardWhoosh: boolean;
194
+ /** The speech track's fade at every cut, in milliseconds (10-100). */
195
+ cutAudioFadeMs: number;
196
+ /**
197
+ * Cross-fade the two sides of every cut over `cutAudioFadeMs`, centred on the cut, instead of fading each side out
198
+ * and in (a short dip). The speech stays where the picture is.
199
+ */
200
+ isCutAudioCrossfade: boolean;
158
201
  }
159
202
 
160
- export const DEFAULT_EDIT: FootageEdit = { zoomPercent: 108, fadeFrames: 10, kenBurnsPercent: 0, isEndCardWhoosh: false };
203
+ export const DEFAULT_EDIT: FootageEdit = { zoomPercent: 108, zoomEaseFrames: 0, fadeFrames: 10, kenBurnsPercent: 0, isEndCardWhoosh: false, cutAudioFadeMs: 20, isCutAudioCrossfade: false };
161
204
 
162
205
  /** The film's edit settings with the defaults filled in. */
163
206
  export function getFootageEdit(film: Pick<FootageFilm, "edit">): FootageEdit {