@softure-ai/marketing-kit 0.35.0 → 0.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/README.md +23 -1
  3. package/dist/config/config.d.ts.map +1 -1
  4. package/dist/config/config.js +1 -0
  5. package/dist/config/config.js.map +1 -1
  6. package/dist/config/schema.d.ts +2 -0
  7. package/dist/config/schema.d.ts.map +1 -1
  8. package/dist/config/schema.js +8 -2
  9. package/dist/config/schema.js.map +1 -1
  10. package/dist/footage/captions.d.ts +22 -2
  11. package/dist/footage/captions.d.ts.map +1 -1
  12. package/dist/footage/captions.js +23 -4
  13. package/dist/footage/captions.js.map +1 -1
  14. package/dist/footage/compose.d.ts +9 -0
  15. package/dist/footage/compose.d.ts.map +1 -1
  16. package/dist/footage/compose.js +35 -14
  17. package/dist/footage/compose.js.map +1 -1
  18. package/dist/footage/draft.d.ts +3 -1
  19. package/dist/footage/draft.d.ts.map +1 -1
  20. package/dist/footage/draft.js +72 -14
  21. package/dist/footage/draft.js.map +1 -1
  22. package/dist/footage/edl.d.ts +26 -0
  23. package/dist/footage/edl.d.ts.map +1 -1
  24. package/dist/footage/edl.js +48 -1
  25. package/dist/footage/edl.js.map +1 -1
  26. package/dist/footage/film.d.ts +25 -2
  27. package/dist/footage/film.d.ts.map +1 -1
  28. package/dist/footage/film.js +10 -1
  29. package/dist/footage/film.js.map +1 -1
  30. package/dist/footage/overlays.d.ts.map +1 -1
  31. package/dist/footage/overlays.js +5 -0
  32. package/dist/footage/overlays.js.map +1 -1
  33. package/dist/footage/picture.d.ts +5 -0
  34. package/dist/footage/picture.d.ts.map +1 -1
  35. package/dist/footage/picture.js +26 -8
  36. package/dist/footage/picture.js.map +1 -1
  37. package/dist/footage/pipeline.d.ts.map +1 -1
  38. package/dist/footage/pipeline.js +7 -3
  39. package/dist/footage/pipeline.js.map +1 -1
  40. package/dist/footage/split-screen.d.ts +167 -0
  41. package/dist/footage/split-screen.d.ts.map +1 -0
  42. package/dist/footage/split-screen.js +233 -0
  43. package/dist/footage/split-screen.js.map +1 -0
  44. package/dist/footage/tools.d.ts +1 -1
  45. package/dist/footage/tools.d.ts.map +1 -1
  46. package/dist/footage/tools.js +1 -1
  47. package/dist/footage/tools.js.map +1 -1
  48. package/dist/index.d.ts +3 -2
  49. package/dist/index.d.ts.map +1 -1
  50. package/dist/index.js +3 -2
  51. package/dist/index.js.map +1 -1
  52. package/package.json +1 -1
  53. package/schema/edl.schema.json +129 -0
  54. package/schema/marketing.schema.json +9 -2
  55. package/src/config/config.ts +1 -0
  56. package/src/config/schema.ts +8 -2
  57. package/src/footage/captions.ts +32 -4
  58. package/src/footage/compose.ts +30 -8
  59. package/src/footage/draft.ts +83 -14
  60. package/src/footage/edl.ts +46 -1
  61. package/src/footage/film.ts +20 -2
  62. package/src/footage/overlays.ts +5 -0
  63. package/src/footage/picture.ts +27 -9
  64. package/src/footage/pipeline.ts +7 -3
  65. package/src/footage/split-screen.ts +319 -0
  66. package/src/footage/tools.ts +1 -1
  67. package/src/index.ts +30 -0
@@ -5,6 +5,7 @@ import { cameraSchema, describeCamera, isSameCamera, type EdlCamera } from "./ca
5
5
  import { overlaySchema, type EdlOverlay } from "./overlay-schema.js";
6
6
  import { DRAWN_TRANSITION_FRAMES, isDrawnTransition, STUTTER_FRAMES } from "./drawn-transitions.js";
7
7
  import { FOOTAGE_FPS } from "./normalize.js";
8
+ import { findSplitCaptionProblems, findSplitLabelProblems, splitSchema, type EdlSplit } from "./split-screen.js";
8
9
  import { CUT_TRANSITIONS, type ClipRole, type CutOptions, type CutTransition, type FootageTransitions } from "./film.js";
9
10
  import { splitFilmSentences, type Sentence } from "./sentences.js";
10
11
  import { isFiller } from "./words.js";
@@ -69,6 +70,8 @@ export interface EdlSegment {
69
70
  audio?: EdlAudio;
70
71
  /** Moves the caption box for the captions that start in this segment; absent: the film's box. */
71
72
  caption?: EdlCaption;
73
+ /** A split screen (`split-screen.ts`): the picture is three shots stacked in bands; absent: the segment's own clip. */
74
+ split?: EdlSplit;
72
75
  }
73
76
 
74
77
  export interface Edl {
@@ -125,6 +128,7 @@ export const edlSchema = z
125
128
  })
126
129
  .optional()
127
130
  .describe("Moves the caption box for the captions that start in this segment (px of the film's frame). Without it, the film's caption box."),
131
+ split: splitSchema.optional(),
128
132
  }),
129
133
  )
130
134
  .min(1, "the cut needs at least one segment")
@@ -294,6 +298,8 @@ export function getSoundSpan(segment: EdlSegment): { clip: string; start: number
294
298
  export interface EdlCaptionBounds {
295
299
  frameHeight: number;
296
300
  cardHeight: number;
301
+ /** The frame's width, for a split screen's labels; absent: a 9:16 frame's (`frameHeight` × 9 / 16). */
302
+ frameWidth?: number;
297
303
  }
298
304
 
299
305
  /**
@@ -319,6 +325,7 @@ export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[],
319
325
  if (segment.end - segment.start < MIN_SEGMENT_SECONDS) problems.push(`${at}: shorter than ${MIN_SEGMENT_SECONDS} s (start ${segment.start}, end ${segment.end})`);
320
326
  if (captionBounds !== undefined) problems.push(...findCaptionProblems(`${at}.caption`, segment.caption, captionBounds));
321
327
  problems.push(...findCameraProblems(at, segment));
328
+ if (segment.split !== undefined) problems.push(...findSplitProblems({ edl, index, split: segment.split, checkSpan, captionBounds }));
322
329
  const { audio } = segment;
323
330
  if (audio === undefined) return;
324
331
  checkSpan(`${at}.audio`, audio);
@@ -330,6 +337,43 @@ export function validateEdl(edl: Edl, clips: Pick<EdlClip, "id" | "duration">[],
330
337
  return problems;
331
338
  }
332
339
 
340
+ /** The 1080 px wide 9:16 frame a split screen's labels are measured in when no bounds name the frame. */
341
+ const SPLIT_CHECK_FRAME = { width: 1080, height: 1920 };
342
+
343
+ /**
344
+ * What makes a split screen unusable: a band's clip missing or too short for the segment, a zoom of its own (a camera,
345
+ * Ken Burns, a zoom or stutter cut: the bands hold still), the cut's last segment (the end card's backdrop is one
346
+ * shot), the segment after it starting on a fade or a drawn transition (its outgoing tail is one shot), a label too
347
+ * wide, and (with `captionBounds`) captions that would cover a label or an arrow.
348
+ */
349
+ function findSplitProblems(input: { edl: Edl; index: number; split: EdlSplit; checkSpan: (at: string, span: { clip: string; start: number; end: number }) => void; captionBounds: EdlCaptionBounds | undefined }): string[] {
350
+ const { edl, index, split, captionBounds } = input;
351
+ const segment = edl.segments[index] as EdlSegment;
352
+ const at = `segments[${String(index)}].split`;
353
+ const length = segment.end - segment.start;
354
+ const problems: string[] = [];
355
+ split.bands.forEach((band, bandIndex) => input.checkSpan(`${at}.bands[${String(bandIndex)}]`, { clip: band.clip, start: band.start, end: r3(band.start + length) }));
356
+ if (segment.camera !== undefined) problems.push(`${at}: a split screen holds still; drop the segment's camera`);
357
+ if (segment.motion === "kenburns") problems.push(`${at}: a split screen holds still; set motion to none`);
358
+ if (segment.transition === "zoom" || segment.transition === "stutter") problems.push(`${at}: a split screen holds still; start it on a cut or a fade, not a ${segment.transition}`);
359
+ const next = edl.segments[index + 1];
360
+ if (next === undefined) problems.push(`${at}: a split screen cannot end the cut (the end card's backdrop is one shot); put a full shot after it`);
361
+ else if (next.transition === "fade" || isDrawnTransition(next.transition)) problems.push(`${at}: the segment after a split screen starts on a ${next.transition}; start "${next.id}" on a cut`);
362
+ const frame = captionBounds === undefined ? SPLIT_CHECK_FRAME : { width: captionBounds.frameWidth ?? Math.round((captionBounds.frameHeight * 9) / 16), height: captionBounds.frameHeight };
363
+ problems.push(...findSplitLabelProblems(split, frame, at));
364
+ if (captionBounds === undefined) return problems;
365
+ const ranges = (edl.captionRanges ?? []).filter((range) => {
366
+ const start = getSegmentStarts(edl)[index] as number;
367
+ return range.from < start + length && start < range.to;
368
+ });
369
+ problems.push(...findSplitCaptionProblems({ at: `segments[${String(index)}]`, caption: segment.caption, frame, cardHeight: captionBounds.cardHeight }));
370
+ for (const range of ranges) {
371
+ const which = (edl.captionRanges ?? []).indexOf(range);
372
+ problems.push(...findSplitCaptionProblems({ at: `captionRanges[${String(which)}]`, caption: range, frame, cardHeight: captionBounds.cardHeight }).map((problem) => problem.replace(`captionRanges[${String(which)}].caption`, `captionRanges[${String(which)}]`)));
373
+ }
374
+ return problems;
375
+ }
376
+
333
377
  /** A camera beside the film's own zoom (a Ken Burns zoom, a zoom cut), which owns the scale too, and a fade length on another cut. */
334
378
  function findCameraProblems(at: string, segment: EdlSegment): string[] {
335
379
  const problems: string[] = [];
@@ -447,7 +491,7 @@ export function diffEdl(before: Edl, after: Edl): string[] {
447
491
  const old = new Map(before.segments.map((segment) => [segment.id, segment]));
448
492
  const next = new Map(after.segments.map((segment) => [segment.id, segment]));
449
493
  for (const segment of before.segments) if (!next.has(segment.id)) lines.push(`dropped ${segment.id}`);
450
- for (const segment of after.segments) if (!old.has(segment.id)) lines.push(`added ${segment.id} (${segment.clip} ${formatSpan(segment)})`);
494
+ for (const segment of after.segments) if (!old.has(segment.id)) lines.push(`added ${segment.id} (${segment.clip} ${formatSpan(segment)}${segment.split === undefined ? "" : `, split screen ${segment.split.bands.map((band) => band.clip).join(", ")}`})`);
451
495
  const common = after.segments.filter((segment) => old.has(segment.id)).map((segment) => segment.id);
452
496
  const places = new Map(before.segments.filter((segment) => next.has(segment.id)).map((segment, index) => [segment.id, index]));
453
497
  const staying = findStayingIds(common, places);
@@ -462,6 +506,7 @@ export function diffEdl(before: Edl, after: Edl): string[] {
462
506
  if (a.transition !== b.transition) lines.push(`${id}: ${a.transition} → ${b.transition}`);
463
507
  if (a.motion !== b.motion) lines.push(`${id}: motion ${a.motion} → ${b.motion}`);
464
508
  if (!isSameCamera(a.camera, b.camera)) lines.push(`${id}: camera ${describeCamera(a.camera)} → ${describeCamera(b.camera)}`);
509
+ if (JSON.stringify(a.split) !== JSON.stringify(b.split)) lines.push(b.split === undefined ? `${id}: one shot again` : `${id}: split screen ${b.split.bands.map((band) => `${band.clip} "${band.label}"`).join(", ")}`);
465
510
  if (a.fadeFrames !== b.fadeFrames) lines.push(`${id}: fade ${a.fadeFrames === undefined ? "default" : `${String(a.fadeFrames)} frames`} → ${b.fadeFrames === undefined ? "default" : `${String(b.fadeFrames)} frames`}`);
466
511
  }
467
512
  for (const segment of after.segments) {
@@ -124,8 +124,14 @@ export interface FootageFilm {
124
124
  music: FootageMusic;
125
125
  /** Captions from the transcript. */
126
126
  captions: boolean;
127
- /** `phrase` (the default): 2-6 words of plain text with a soft shadow; `highlight`: a pill that lights each word as it is said. */
127
+ /**
128
+ * `phrase` (the default): 2-6 words of plain text with a soft shadow; `highlight`: a pill that lights each word as it
129
+ * is said; `ribbon`: bold capitals, each line on its own flat strip of `captionBackground`, the block turned by
130
+ * `captionRibbonTilt`.
131
+ */
128
132
  captionStyle?: CaptionStyle;
133
+ /** A `ribbon` caption's turn in degrees, counter-clockwise positive (-10 to 10); absent: `CAPTION_RIBBON_TILT`. */
134
+ captionRibbonTilt?: number;
129
135
  /**
130
136
  * How a `highlight` caption lights the word being said: `text` (the default) colours its text `captionHighlight`;
131
137
  * `box` puts it on a rounded box of `captionHighlightBox` with its text in `captionHighlightText` (both brand colours
@@ -282,10 +288,22 @@ export const DEFAULT_CUT: Omit<CutOptions, "fillers"> = { pauseSeconds: 0.6, pad
282
288
  /** Hard cuts everywhere: an ordinary phone video, not a promo. */
283
289
  export const DEFAULT_TRANSITIONS: FootageTransitions = { jump: "cut", clip: "cut" };
284
290
 
285
- export const CAPTION_STYLES = ["phrase", "highlight"] as const;
291
+ export const CAPTION_STYLES = ["phrase", "highlight", "ribbon"] as const;
286
292
 
287
293
  export type CaptionStyle = (typeof CAPTION_STYLES)[number];
288
294
 
295
+ /** A `ribbon` caption's turn in degrees, counter-clockwise positive as ASS turns, unless the film names its own. */
296
+ export const CAPTION_RIBBON_TILT = 4;
297
+
298
+ /** The most a `ribbon` caption may turn either way, in degrees. */
299
+ export const CAPTION_RIBBON_MAX_TILT = 10;
300
+
301
+ /**
302
+ * A `ribbon` caption's geometry, shared by the render and the draft: its lines' height in font sizes, each line's strip
303
+ * padding in font sizes, the strip's corner in px of a 1080 px frame, and the letters' weight.
304
+ */
305
+ export const CAPTION_RIBBON = { lineHeight: 1.42, padding: { y: 0.1, x: 0.32 }, radius: 6, weight: 800 } as const;
306
+
289
307
  /** How a `highlight` caption lights the word being said: its text colour, a box behind it (issue #1199), or an accent underline. */
290
308
  export const CAPTION_HIGHLIGHT_STYLES = ["text", "box", "underline"] as const;
291
309
 
@@ -518,6 +518,11 @@ export function resolveEdlOverlays(input: ResolveOverlaysInput): ResolvedOverlay
518
518
  const segment = findSegmentAt(edl, start);
519
519
  const segmentStart = starts[segment] as number;
520
520
  const segmentId = edl.segments[segment]?.id ?? "";
521
+ const split = edl.segments.findIndex((candidate, i) => candidate.split !== undefined && (starts[i] as number) < end - 0.001 && start < (starts[i] as number) + candidate.end - candidate.start - 0.001);
522
+ if (split !== -1) {
523
+ problems.push(`${at}: segment "${edl.segments[split]?.id ?? ""}" is a split screen with its own labels and arrows; keep overlays off it`);
524
+ return;
525
+ }
521
526
  if (!isTitleBand(overlay.kind)) {
522
527
  const shot = findShotSpan(edl, starts, { segment, end, isPunchIn: overlay.kind === "punch_in" });
523
528
  if (end > shot.end + 0.01) {
@@ -4,6 +4,7 @@ import { buildDrawnTransitionFilters, FRAME_TIMING, getTransitionOverlapFrames,
4
4
  import { getEdlDuration, getSegmentStarts, type Edl } from "./edl.js";
5
5
  import { getFootageEdit, type FootageFilm } from "./film.js";
6
6
  import { FOOTAGE_FPS, getThreadArgs, type Frame, type NormalizeEncoder } from "./normalize.js";
7
+ import { buildSplitScreenFilters, getSplitLayerInputArgs, type EdlSplit } from "./split-screen.js";
7
8
 
8
9
  /**
9
10
  * The cut's picture in film time, which the render's composition plays as one `<video>`: one short ffmpeg per segment
@@ -50,6 +51,8 @@ export interface PicturePiece {
50
51
  frames: number;
51
52
  /** For a `fade` cut or a drawn transition, the outgoing tail the piece starts over; null for any other cut. */
52
53
  fade: PictureFadeTail | null;
54
+ /** The segment's split screen: the piece is its three bands under its layer (`split-screen.ts`); absent: `clip`. */
55
+ split?: EdlSplit;
53
56
  }
54
57
 
55
58
  /** The film frame a film second falls on. */
@@ -86,7 +89,7 @@ export function planFootagePicture(film: Pick<FootageFilm, "edit">, edl: Edl): P
86
89
  const scale = isCamera ? r4(Math.max(1, reached / (scales[index] as number))) : reached;
87
90
  fade = { ...(kind === "fade" ? {} : { kind }), clip: previous.clip, start: r3(previous.end), frames: fadeFrames, scale, origin: getCameraOrigin(previous.camera) };
88
91
  }
89
- return { clip: segment.clip, start: r3(segment.start), frame, frames, fade };
92
+ return { clip: segment.clip, start: r3(segment.start), frame, frames, fade, ...(segment.split === undefined ? {} : { split: segment.split }) };
90
93
  });
91
94
  }
92
95
 
@@ -133,6 +136,8 @@ export interface PicturePieceArgsInput {
133
136
  output: string;
134
137
  /** The intermediate clips' encoder (`RenderFootageOptions.normalize`): its preset and threads encode the piece too. */
135
138
  encoder?: NormalizeEncoder;
139
+ /** A split piece's layer (`renderSplitLayerPng` at `frame`'s size); needed when the piece has a `split`. */
140
+ splitLayer?: string;
136
141
  }
137
142
 
138
143
  /** One piece's ffmpeg arguments: its clip (and a fade's outgoing tail), exactly `frames` frames, H.264 without sound. */
@@ -145,19 +150,32 @@ export function buildPicturePieceArgs(input: PicturePieceArgsInput): string[] {
145
150
  };
146
151
  const threads = getThreadArgs(input.encoder?.threads, "normalize.threads");
147
152
  const seconds = (frames: number): string => String(r3(frames / FOOTAGE_FPS + 1 / FOOTAGE_FPS));
148
- const inputs = [...threads.output, "-ss", String(piece.start), "-t", seconds(piece.frames), "-i", getClip(piece.clip)];
149
- let graph = `[0:v]${fitStream(piece.frames).join(",")}[picture]`;
153
+ const inputs: string[] = [];
154
+ const prelude: string[] = [];
155
+ let source = `[0:v]${fitStream(piece.frames).join(",")}`;
156
+ if (piece.split === undefined) {
157
+ inputs.push(...threads.output, "-ss", String(piece.start), "-t", seconds(piece.frames), "-i", getClip(piece.clip));
158
+ } else {
159
+ if (input.splitLayer === undefined) throw new Error("A split piece needs its layer (splitLayer).");
160
+ for (const band of piece.split.bands) inputs.push(...threads.output, "-ss", String(r3(band.start)), "-t", seconds(piece.frames), "-i", getClip(band.clip));
161
+ inputs.push(...getSplitLayerInputArgs(input.splitLayer, Number(seconds(piece.frames)), FOOTAGE_FPS));
162
+ prelude.push(...buildSplitScreenFilters({ inputs: ["0:v", "1:v", "2:v"], layer: "3:v", split: piece.split, frame: input.frame, fit: fitStream(piece.frames), output: "split", prefix: "sp" }));
163
+ source = "[split]null";
164
+ }
165
+ const tailInput = piece.split === undefined ? 1 : 4;
166
+ let graph = `${source}[picture]`;
150
167
  if (piece.fade !== null) {
151
168
  const { fade } = piece;
152
169
  inputs.push(...threads.output, "-ss", String(fade.start), "-t", seconds(fade.frames), "-i", getClip(fade.clip));
153
170
  const tail = [...zoomFilters(fade.scale, input.frame, fade.origin ?? CAMERA_ORIGIN), ...fitStream(fade.frames)];
154
171
  graph = fade.kind === undefined ? [
155
- `[0:v]${fitStream(piece.frames).join(",")}[incoming]`,
156
- `[1:v]${tail.join(",")}[outgoing]`,
172
+ `${source}[incoming]`,
173
+ `[${String(tailInput)}:v]${tail.join(",")}[outgoing]`,
157
174
  // xfade's output is the outgoing tail plus the incoming piece minus the fade: the piece's own length.
158
175
  `[outgoing][incoming]xfade=transition=fade:duration=${r3(fade.frames / FOOTAGE_FPS)}:offset=0,trim=end_frame=${piece.frames}[picture]`,
159
- ].join(";") : buildDrawnPieceGraph({ kind: fade.kind, piece, tail, frame: input.frame });
176
+ ].join(";") : buildDrawnPieceGraph({ kind: fade.kind, piece, tail, frame: input.frame, source, tailInput });
160
177
  }
178
+ if (prelude.length > 0) graph = [...prelude, graph].join(";");
161
179
  return [
162
180
  "-v",
163
181
  "error",
@@ -196,14 +214,14 @@ export function buildPicturePieceArgs(input: PicturePieceArgsInput): string[] {
196
214
  * A piece that starts on a drawn transition: its first `frames` blended with the outgoing tail, then the rest of its
197
215
  * own frames (the plan gives a drawn transition only to a piece longer than it).
198
216
  */
199
- function buildDrawnPieceGraph(input: { kind: DrawnTransition; piece: PicturePiece; tail: string[]; frame: Frame }): string {
217
+ function buildDrawnPieceGraph(input: { kind: DrawnTransition; piece: PicturePiece; tail: string[]; frame: Frame; source: string; tailInput: number }): string {
200
218
  const { piece } = input;
201
219
  const frames = piece.fade?.frames ?? 0;
202
220
  return [
203
- `[0:v]${fitStream(piece.frames).join(",")},split[full1][full2]`,
221
+ `${input.source},split[full1][full2]`,
204
222
  `[full1]trim=end_frame=${frames},setpts=PTS-STARTPTS,${FRAME_TIMING}[incoming]`,
205
223
  `[full2]trim=start_frame=${frames},setpts=PTS-STARTPTS,${FRAME_TIMING}[rest]`,
206
- `[1:v]${input.tail.join(",")}[outgoing]`,
224
+ `[${String(input.tailInput)}:v]${input.tail.join(",")}[outgoing]`,
207
225
  ...buildDrawnTransitionFilters({ kind: input.kind, outgoing: "outgoing", incoming: "incoming", output: "blend", frame: input.frame, prefix: "dt" }),
208
226
  `[blend][rest]concat=n=2:v=1:a=0,settb=1/${FOOTAGE_FPS},fps=${FOOTAGE_FPS}[picture]`,
209
227
  ].join(";");
@@ -18,7 +18,7 @@ import { composeFootage, getFootageFilmSeconds, getFootageSfxCues } from "./comp
18
18
  import { getCaptionHighlight } from "./caption-highlight.js";
19
19
  import { getOverlayTextsByRole, isHandFontNeeded, resolveEdlOverlays } from "./overlays.js";
20
20
  import { HAND_FONT } from "./hand-font.js";
21
- import { findMissingCaptionGlyphs, getCaptionBounds, getFootageCaptionLayout } from "./captions.js";
21
+ import { findMissingCaptionGlyphs, getCaptionBounds, getFootageCaptionLayout, toCaptionDisplayText } from "./captions.js";
22
22
  import { err, getProviderFootageFailure, ok, type FootageResult } from "./failure.js";
23
23
  import { findLowDiskProblems, getFreeBytes, getRenderTempDir, type FreeBytesReader } from "./disk.js";
24
24
  import { buildEdl, getEdlDuration, readEdl, remapWords, validateEdl, type Edl } from "./edl.js";
@@ -50,6 +50,7 @@ import {
50
50
  } from "./loudness.js";
51
51
  import { duckingKeyframes, getDuckingOptions } from "./music.js";
52
52
  import { FOOTAGE_FPS, normalizeFootage, type Frame, type NormalizeEncoder } from "./normalize.js";
53
+ import { writeSplitLayers } from "./split-screen.js";
53
54
  import { buildPictureConcatArgs, buildPictureList, buildPicturePieceArgs, getPictureFrames, planFootagePicture } from "./picture.js";
54
55
  import { describeProbe, getTrimmedDuration, probeFootage, type FootageProbe } from "./probe.js";
55
56
  import {
@@ -865,7 +866,7 @@ async function runRenderFootage(
865
866
  const layout = { ...base, caption: getFootageCaptionLayout(film, options.layout ?? {}) };
866
867
  const missing = findMissingCaptionGlyphs({
867
868
  fonts: options.brand.fonts,
868
- captions: film.captions ? words.map((word) => word.text) : [],
869
+ captions: film.captions ? words.map((word) => toCaptionDisplayText(film, word.text, options.brand.locale)) : [],
869
870
  overlays: getOverlayTextsByRole(overlays.overlays, options.brand.locale),
870
871
  endCard: getEndCardTexts(film.endCard),
871
872
  brandName: options.brand.name,
@@ -1169,10 +1170,13 @@ async function buildPicture(input: BuildPictureInput): Promise<void> {
1169
1170
  mkdirSync(dir, { recursive: true });
1170
1171
  const aborting = input.signal === undefined ? {} : { signal: input.signal };
1171
1172
  const files: string[] = [];
1173
+ const frame = getFrame(input.film);
1174
+ const layers = writeSplitLayers(input.edl.segments, frame, dir);
1172
1175
  for (const [index, piece] of pieces.entries()) {
1173
1176
  const file = join(dir, `${String(index).padStart(3, "0")}.mp4`);
1174
1177
  const done = piece.frame / FOOTAGE_FPS;
1175
- await runFfmpeg(`The picture of segment ${index + 1}`, buildPicturePieceArgs({ piece, clips: input.clips, frame: getFrame(input.film), output: file, ...(input.encoder === undefined ? {} : { encoder: input.encoder }) }), {
1178
+ const layer = layers[input.edl.segments[index]?.id ?? ""];
1179
+ await runFfmpeg(`The picture of segment ${index + 1}`, buildPicturePieceArgs({ piece, clips: input.clips, frame, output: file, ...(input.encoder === undefined ? {} : { encoder: input.encoder }), ...(layer === undefined ? {} : { splitLayer: layer }) }), {
1176
1180
  ...aborting,
1177
1181
  output: file,
1178
1182
  onSeconds: (seconds) => input.onSeconds(done + seconds),
@@ -0,0 +1,319 @@
1
+ import { writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+
4
+ import { Resvg } from "@resvg/resvg-js";
5
+ import { z } from "zod";
6
+
7
+ import { HAND_FONT, measureHandText } from "./hand-font.js";
8
+ import { FOOTAGE_FPS, type Frame } from "./normalize.js";
9
+
10
+ /**
11
+ * A split screen: a segment's picture made of three shots stacked in horizontal bands, as reels compare three places.
12
+ * Each band plays its own clip from its own second for the segment's length, cropped to the band (a `focus` picks how
13
+ * high), and carries a short handwritten label with a drawn curved arrow pointing into it. The segment's sound stays
14
+ * its own (`clip`, or `audio`).
15
+ *
16
+ * The draft (`draft.ts`) and the render's picture (`picture.ts`) build the bands with one filter builder
17
+ * (`buildSplitScreenFilters`) and lay the same layer over them: the labels, the arrows and the thin lines between the
18
+ * bands drawn once as an SVG (`getSplitLayerSvg`) and rasterised by resvg at the path's frame size
19
+ * (`renderSplitLayerPng`), so both paths show the same picture. The handwriting is the kit's own hand font (Caveat,
20
+ * OFL); the arrows are curves made in code.
21
+ *
22
+ * The labels zig-zag: the top band's sits low on the left with its arrow rising into the shot, the middle band's high
23
+ * on the right, the bottom band's high on the left with arrows falling into theirs. That keeps the frame's top (the
24
+ * platform's header) and its foot (the platform's buttons) clear and leaves the lower half of the middle band free for
25
+ * the captions (`getSplitCaption`, `findSplitCaptionProblems`).
26
+ */
27
+
28
+ export const SPLIT_LAYOUTS = ["split3"] as const;
29
+
30
+ export type SplitLayout = (typeof SPLIT_LAYOUTS)[number];
31
+
32
+ /** How many bands a `split3` has. */
33
+ export const SPLIT_BANDS = 3;
34
+
35
+ /** The longest label, in characters; a label must also fit its width (`MAX_LABEL_WIDTH_SHARE` of the frame). */
36
+ export const MAX_SPLIT_LABEL_CHARACTERS = 24;
37
+
38
+ /** A label's handwriting size, in px of a 1080 px wide frame. */
39
+ const LABEL_SIZE = 60;
40
+ /** The widest a label may be, as a share of the frame's width (the arrow needs the rest). */
41
+ const MAX_LABEL_WIDTH_SHARE = 0.5;
42
+ /** A label's distance from the frame's side, in px of a 1080 px wide frame. */
43
+ const LABEL_SIDE = 64;
44
+ /** The arrow's stroke and its head's strokes, in px of a 1080 px wide frame. */
45
+ const ARROW_STROKE = 6;
46
+ const ARROW_HEAD = 30;
47
+ /** The line between two bands, in px of a 1080 px wide frame. */
48
+ const BAND_LINE = 4;
49
+ /** How far a label's shadow falls and blurs, in px of a 1080 px wide frame. */
50
+ const SHADOW_OFFSET = 3;
51
+ const SHADOW_BLUR = 4;
52
+
53
+ /** One band of a split screen: its clip, the second it starts from, its label, and how high the crop sits. */
54
+ export interface EdlSplitBand {
55
+ clip: string;
56
+ /** Seconds in the trimmed clip where the band's shot starts; it plays for the segment's length. */
57
+ start: number;
58
+ /** A short handwritten label (one line, at most `MAX_SPLIT_LABEL_CHARACTERS`). */
59
+ label: string;
60
+ /** Where the band's crop sits in the shot's height, 0 (its top) to 1 (its foot); absent: 0.5, the middle. */
61
+ focus?: number;
62
+ }
63
+
64
+ /** A segment's split screen (`EdlSegment.split`). */
65
+ export interface EdlSplit {
66
+ layout: SplitLayout;
67
+ bands: [EdlSplitBand, EdlSplitBand, EdlSplitBand];
68
+ }
69
+
70
+ const bandSchema = z.strictObject({
71
+ clip: z.string().min(1).describe("The band's clip id."),
72
+ start: z.number().min(0).describe("Seconds in that trimmed clip where the band's shot starts; it plays for the segment's length."),
73
+ label: z
74
+ .string()
75
+ .trim()
76
+ .min(1, "a band needs a label")
77
+ .max(MAX_SPLIT_LABEL_CHARACTERS, `a label is at most ${String(MAX_SPLIT_LABEL_CHARACTERS)} characters`)
78
+ .regex(/^[^\n\r]*$/, "a label is one line")
79
+ .describe(`A short handwritten label, one line, at most ${String(MAX_SPLIT_LABEL_CHARACTERS)} characters (a place's name).`),
80
+ focus: z.number().min(0).max(1).optional().describe("Where the band's crop sits in the shot's height: 0 its top, 1 its foot; 0.5 (the middle) without it."),
81
+ });
82
+
83
+ export const splitSchema = z
84
+ .strictObject({
85
+ layout: z.enum(SPLIT_LAYOUTS).describe("split3: three shots stacked in horizontal bands."),
86
+ bands: z.tuple([bandSchema, bandSchema, bandSchema]).describe("The three bands, top to bottom."),
87
+ })
88
+ .describe(
89
+ "A split screen: the segment's picture is three shots stacked in bands, each with a handwritten label and a drawn arrow; the segment's sound stays its own. Still (no camera, Ken Burns, zoom or stutter), not the cut's last segment, and the segment after it starts on a plain cut.",
90
+ );
91
+
92
+ /** A rectangle of the frame, in px. */
93
+ export interface SplitBox {
94
+ x: number;
95
+ y: number;
96
+ width: number;
97
+ height: number;
98
+ }
99
+
100
+ /** Where one band sits and where its label and arrow go, in px of the frame. */
101
+ export interface SplitBandPlace {
102
+ /** The band's rectangle (full width). */
103
+ band: SplitBox;
104
+ /** The label's side: its text starts at `x` (left) or ends at it (right). */
105
+ side: "left" | "right";
106
+ /** The label's baseline. */
107
+ labelX: number;
108
+ labelY: number;
109
+ /** Where the arrow points, in the band's shot. */
110
+ arrowTo: { x: number; y: number };
111
+ /** The stretch of the frame's height the label and its arrow take (captions keep off it). */
112
+ reserved: { top: number; bottom: number };
113
+ }
114
+
115
+ const round = (value: number): number => Math.round(value);
116
+
117
+ /** An even size of at least 2 px, as yuv420p needs. */
118
+ function toEven(value: number): number {
119
+ return Math.max(2, 2 * Math.floor(value / 2));
120
+ }
121
+
122
+ /** The three bands' heights: equal and even, the last one taking what rounding leaves. */
123
+ export function getSplitBandHeights(frame: Frame): [number, number, number] {
124
+ const band = toEven(frame.height / SPLIT_BANDS);
125
+ return [band, band, frame.height - 2 * band];
126
+ }
127
+
128
+ /**
129
+ * Each band's rectangle, label place and arrow target. The top band's label sits at 78 % of its height on the left,
130
+ * its arrow rising to the shot's middle; the middle band's at 24 % on the right and the bottom band's at 22 % on the
131
+ * left, their arrows falling into the shot.
132
+ */
133
+ export function getSplitPlaces(frame: Frame): [SplitBandPlace, SplitBandPlace, SplitBandPlace] {
134
+ const scale = frame.width / 1080;
135
+ const heights = getSplitBandHeights(frame);
136
+ const tops = [0, heights[0], heights[0] + heights[1]];
137
+ const side = round(LABEL_SIDE * scale);
138
+ const label = LABEL_SIZE * scale;
139
+ const place = (index: 0 | 1 | 2): SplitBandPlace => {
140
+ const top = tops[index] as number;
141
+ const height = heights[index];
142
+ const band = { x: 0, y: top, width: frame.width, height };
143
+ if (index === 0) {
144
+ const labelY = round(top + 0.78 * height);
145
+ const arrowTo = { x: round(0.56 * frame.width), y: round(top + 0.42 * height) };
146
+ return { band, side: "left", labelX: side, labelY, arrowTo, reserved: { top: arrowTo.y, bottom: round(labelY + 0.35 * label) } };
147
+ }
148
+ const labelY = round(top + (index === 1 ? 0.24 : 0.22) * height);
149
+ const arrowTo = { x: round((index === 1 ? 0.42 : 0.58) * frame.width), y: round(top + (index === 1 ? 0.46 : 0.42) * height) };
150
+ return { band, side: index === 1 ? "right" : "left", labelX: index === 1 ? frame.width - side : side, labelY, arrowTo, reserved: { top: round(labelY - label), bottom: arrowTo.y } };
151
+ };
152
+ return [place(0), place(1), place(2)];
153
+ }
154
+
155
+ /** How wide `label` is written, in px of the frame. */
156
+ function measureLabel(label: string, frame: Frame): number {
157
+ return measureHandText(label, LABEL_SIZE * (frame.width / 1080));
158
+ }
159
+
160
+ /** What makes a split's labels unusable in this frame: a label too wide for its band (empty when they all fit). */
161
+ export function findSplitLabelProblems(split: EdlSplit, frame: Frame, at: string): string[] {
162
+ const limit = round(frame.width * MAX_LABEL_WIDTH_SHARE);
163
+ return split.bands.flatMap((band, index) => {
164
+ const width = measureLabel(band.label, frame);
165
+ return width > limit ? [`${at}.bands[${String(index)}].label: "${band.label}" is ${String(width)} px wide, more than ${String(limit)} px; shorten it`] : [];
166
+ });
167
+ }
168
+
169
+ const f1 = (value: number): string => String(Math.round(value * 10) / 10);
170
+
171
+ /**
172
+ * The arrow from the label's end to the band's shot: a curve bowed to one side, slightly uneven like a hand's stroke,
173
+ * and its head's two strokes along the curve's last direction.
174
+ */
175
+ export function getSplitArrowPaths(from: { x: number; y: number }, to: { x: number; y: number }, scale: number): { curve: string; head: string } {
176
+ const dx = to.x - from.x;
177
+ const dy = to.y - from.y;
178
+ const length = Math.hypot(dx, dy) || 1;
179
+ // A normal that bows the curve away from the frame's centre line, a third of the way and two thirds unevenly.
180
+ const nx = -dy / length;
181
+ const ny = dx / length;
182
+ const bow = length * 0.22;
183
+ const c1 = { x: from.x + dx * 0.3 + nx * bow, y: from.y + dy * 0.3 + ny * bow };
184
+ const c2 = { x: from.x + dx * 0.72 + nx * bow * 0.8, y: from.y + dy * 0.72 + ny * bow * 0.8 };
185
+ const tx = to.x - c2.x;
186
+ const ty = to.y - c2.y;
187
+ const angle = Math.atan2(ty, tx);
188
+ const head = ARROW_HEAD * scale;
189
+ const wing = (turn: number): string => `M${f1(to.x - head * Math.cos(angle + turn))} ${f1(to.y - head * Math.sin(angle + turn))} L${f1(to.x)} ${f1(to.y)}`;
190
+ return {
191
+ curve: `M${f1(from.x)} ${f1(from.y)} C${f1(c1.x)} ${f1(c1.y)} ${f1(c2.x)} ${f1(c2.y)} ${f1(to.x)} ${f1(to.y)}`,
192
+ head: `${wing(0.5)} ${wing(-0.45)}`,
193
+ };
194
+ }
195
+
196
+ const escapeXml = (text: string): string => text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
197
+
198
+ /** The layer over the bands, as an SVG of the frame's size: the lines between the bands, each label and its arrow. */
199
+ export function getSplitLayerSvg(split: EdlSplit, frame: Frame): string {
200
+ const scale = frame.width / 1080;
201
+ const places = getSplitPlaces(frame);
202
+ const line = BAND_LINE * scale;
203
+ const size = LABEL_SIZE * scale;
204
+ const gap = 18 * scale;
205
+ const parts: string[] = [];
206
+ for (const place of places.slice(1)) {
207
+ parts.push(`<rect x="0" y="${f1(place.band.y - line / 2)}" width="${String(frame.width)}" height="${f1(line)}" fill="#111111"/>`);
208
+ }
209
+ split.bands.forEach((band, index) => {
210
+ const place = places[index] as SplitBandPlace;
211
+ const width = measureLabel(band.label, frame);
212
+ const isLeft = place.side === "left";
213
+ // The arrow starts beside the label's end, at its x-height, and points into the shot.
214
+ const from = { x: isLeft ? place.labelX + width + gap : place.labelX - width - gap, y: place.labelY - (index === 0 ? size * 0.55 : size * 0.2) };
215
+ const arrow = getSplitArrowPaths(from, place.arrowTo, scale);
216
+ parts.push(
217
+ `<g filter="url(#shadow)">`,
218
+ `<text x="${f1(place.labelX)}" y="${f1(place.labelY)}" text-anchor="${isLeft ? "start" : "end"}" font-family="${HAND_FONT.family}" font-weight="600" font-size="${f1(size)}" fill="#ffffff">${escapeXml(band.label)}</text>`,
219
+ `<path d="${arrow.curve}" fill="none" stroke="#ffffff" stroke-width="${f1(ARROW_STROKE * scale)}" stroke-linecap="round"/>`,
220
+ `<path d="${arrow.head}" fill="none" stroke="#ffffff" stroke-width="${f1(ARROW_STROKE * scale)}" stroke-linecap="round" stroke-linejoin="round"/>`,
221
+ `</g>`,
222
+ );
223
+ });
224
+ const shadow = `<filter id="shadow" x="-10%" y="-10%" width="120%" height="120%"><feDropShadow dx="0" dy="${f1(SHADOW_OFFSET * scale)}" stdDeviation="${f1(SHADOW_BLUR * scale)}" flood-color="#000000" flood-opacity="0.55"/></filter>`;
225
+ return `<svg xmlns="http://www.w3.org/2000/svg" width="${String(frame.width)}" height="${String(frame.height)}" viewBox="0 0 ${String(frame.width)} ${String(frame.height)}"><defs>${shadow}</defs>${parts.join("")}</svg>`;
226
+ }
227
+
228
+ /** The layer as a transparent PNG of the frame's size, drawn by resvg with the kit's hand font only. */
229
+ export function renderSplitLayerPng(split: EdlSplit, frame: Frame): Buffer {
230
+ const svg = getSplitLayerSvg(split, frame);
231
+ return new Resvg(svg, { fitTo: { mode: "original" }, font: { loadSystemFonts: false, fontFiles: [HAND_FONT.file], defaultFontFamily: HAND_FONT.family } }).render().asPng();
232
+ }
233
+
234
+ /**
235
+ * Writes each split segment's layer into `dir` at `frame`'s size (`split-<segment id>.png`) and answers the file
236
+ * names by segment id, joined to `dir` (or bare with `isRelative`, for an ffmpeg run in `dir`). None without a split.
237
+ */
238
+ export function writeSplitLayers(segments: readonly { id: string; split?: EdlSplit | undefined }[], frame: Frame, dir: string, isRelative = false): Record<string, string> {
239
+ const files: Record<string, string> = {};
240
+ for (const segment of segments) {
241
+ if (segment.split === undefined) continue;
242
+ const name = `split-${segment.id}.png`;
243
+ writeFileSync(join(dir, name), renderSplitLayerPng(segment.split, frame));
244
+ files[segment.id] = isRelative ? name : join(dir, name);
245
+ }
246
+ return files;
247
+ }
248
+
249
+ export interface SplitScreenFiltersInput {
250
+ /** The three bands' input labels (their clips, already seeked to the band's start). */
251
+ inputs: readonly [string, string, string];
252
+ /** The layer's input label (`renderSplitLayerPng` looped as a video). */
253
+ layer: string;
254
+ split: EdlSplit;
255
+ frame: Frame;
256
+ /** Filters each band runs after its crop (frame rate, length), the same for all three. */
257
+ fit: readonly string[];
258
+ /** The label the composed picture goes out on (yuv420p, the frame's size). */
259
+ output: string;
260
+ /** A prefix for the graph's inner labels, unique in the graph. */
261
+ prefix: string;
262
+ }
263
+
264
+ /**
265
+ * The filter chains (to join with `;`) that crop each shot to its band (scaled to cover it, the crop at its `focus`),
266
+ * stack the three and lay the layer over them.
267
+ */
268
+ export function buildSplitScreenFilters(input: SplitScreenFiltersInput): string[] {
269
+ const { frame, prefix: p } = input;
270
+ const heights = getSplitBandHeights(frame);
271
+ const bands = input.split.bands.map((band, index) => {
272
+ const height = heights[index] as number;
273
+ const focus = band.focus ?? 0.5;
274
+ const chain = [
275
+ `scale=${String(frame.width)}:${String(height)}:force_original_aspect_ratio=increase`,
276
+ `crop=${String(frame.width)}:${String(height)}:(iw-ow)/2:(ih-oh)*${String(focus)}`,
277
+ "setsar=1",
278
+ ...input.fit,
279
+ ];
280
+ return `[${input.inputs[index] as string}]${chain.join(",")}[${p}b${String(index)}]`;
281
+ });
282
+ return [
283
+ ...bands,
284
+ `[${p}b0][${p}b1][${p}b2]vstack=inputs=3[${p}stack]`,
285
+ `[${input.layer}]format=rgba[${p}layer]`,
286
+ `[${p}stack][${p}layer]overlay=0:0:format=auto:shortest=1,format=yuv420p,setsar=1,settb=1/${String(FOOTAGE_FPS)},setpts=N,fps=${String(FOOTAGE_FPS)}[${input.output}]`,
287
+ ];
288
+ }
289
+
290
+ /** The ffmpeg input of a layer PNG: one image looped at the film's rate for `seconds`. */
291
+ export function getSplitLayerInputArgs(file: string, seconds: number, fps: number): string[] {
292
+ return ["-loop", "1", "-framerate", String(fps), "-t", String(seconds), "-i", file];
293
+ }
294
+
295
+ /**
296
+ * The caption box a split segment takes so its captions sit in the lower half of the middle band, clear of every
297
+ * label and arrow: its foot at 94 % of the middle band's height, in px above the frame's foot.
298
+ */
299
+ export function getSplitCaption(frame: Frame): { bottom: number } {
300
+ const places = getSplitPlaces(frame);
301
+ const middle = places[1].band;
302
+ return { bottom: frame.height - round(middle.y + 0.94 * middle.height) };
303
+ }
304
+
305
+ /**
306
+ * Whether a caption box (its foot or its top, and the height of a card of `lines` lines) would cover a split's label
307
+ * or arrow: the stretch of the frame's height it takes against every band's reserved stretch. Empty when clear.
308
+ */
309
+ export function findSplitCaptionProblems(input: { at: string; caption: { bottom?: number; top?: number } | undefined; frame: Frame; cardHeight: number; lines?: number }): string[] {
310
+ const { caption, frame } = input;
311
+ const at = `${input.at}.caption`;
312
+ if (caption?.bottom === undefined && caption?.top === undefined) return [`${at}: a split screen sets its own caption box, clear of its labels (bottom ${String(getSplitCaption(frame).bottom)} px puts the captions in the middle band)`];
313
+ const height = input.cardHeight * (input.lines ?? 2);
314
+ const foot = caption.bottom === undefined ? (caption.top as number) + height : frame.height - caption.bottom;
315
+ const top = caption.bottom === undefined ? (caption.top as number) : foot - height;
316
+ const covered = getSplitPlaces(frame).findIndex((place) => top < place.reserved.bottom && place.reserved.top < foot);
317
+ if (covered === -1) return [];
318
+ return [`${at}: the captions (${String(round(top))}-${String(round(foot))} px) would cover band ${String(covered + 1)}'s label or arrow; bottom ${String(getSplitCaption(frame).bottom)} px puts them in the middle band`];
319
+ }
@@ -36,7 +36,7 @@ export interface FootageToolsOptions {
36
36
  export const REQUIRED_DECODERS = ["hevc", "h264"] as const;
37
37
  export const REQUIRED_FILTERS = ["zscale", "tonemap", "lut3d"] as const;
38
38
  /** What the fast draft needs on top: libass (with libfreetype) for the captions, `xfade` for fades, `zoompan` for Ken Burns. */
39
- export const DRAFT_FILTERS = ["ass", "xfade", "zoompan", "gblur", "geq", "rotate", "blend", "noise", "interleave", "hstack"] as const;
39
+ export const DRAFT_FILTERS = ["ass", "xfade", "zoompan", "gblur", "geq", "rotate", "blend", "noise", "interleave", "hstack", "vstack"] as const;
40
40
 
41
41
  const DRAFT_INSTALL_HINT = "install an ffmpeg built with libass and libfreetype (macOS: brew install ffmpeg, Debian/Ubuntu: apt install ffmpeg)";
42
42
  const INSTALL_HINT = "install an ffmpeg built with libzimg (macOS: brew install ffmpeg, Debian/Ubuntu: apt install ffmpeg)";