@softure-ai/marketing-kit 0.36.2 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -0
- package/README.md +27 -4
- package/dist/footage/draft-overlays.d.ts.map +1 -1
- package/dist/footage/draft-overlays.js +46 -3
- package/dist/footage/draft-overlays.js.map +1 -1
- package/dist/footage/draft.d.ts +94 -6
- package/dist/footage/draft.d.ts.map +1 -1
- package/dist/footage/draft.js +227 -59
- package/dist/footage/draft.js.map +1 -1
- package/dist/footage/overlay-html.d.ts.map +1 -1
- package/dist/footage/overlay-html.js +26 -4
- package/dist/footage/overlay-html.js.map +1 -1
- package/dist/footage/overlay-schema.d.ts +2 -2
- package/dist/footage/overlay-schema.js +4 -4
- package/dist/footage/overlay-schema.js.map +1 -1
- package/dist/footage/overlays.d.ts +66 -1
- package/dist/footage/overlays.d.ts.map +1 -1
- package/dist/footage/overlays.js +126 -27
- package/dist/footage/overlays.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/package.json +2 -2
- package/schema/edl.schema.json +2 -2
- package/src/footage/draft-overlays.ts +58 -2
- package/src/footage/draft.ts +335 -57
- package/src/footage/overlay-html.ts +30 -3
- package/src/footage/overlay-schema.ts +4 -4
- package/src/footage/overlays.ts +127 -22
- package/src/index.ts +19 -0
package/src/footage/draft.ts
CHANGED
|
@@ -45,6 +45,7 @@ import {
|
|
|
45
45
|
type PreparedMix,
|
|
46
46
|
type TranscribeOutcome,
|
|
47
47
|
} from "./pipeline.js";
|
|
48
|
+
import { buildPictureList } from "./picture.js";
|
|
48
49
|
import { runFfmpeg, withFfmpegFailureReporting } from "./process.js";
|
|
49
50
|
import { buildMusicArgs, buildSpeechArgs, getSpeechCutOptions } from "./render.js";
|
|
50
51
|
|
|
@@ -639,9 +640,8 @@ function buildPictureFilters(plan: DraftSegmentPlan, draft: Frame): string[] {
|
|
|
639
640
|
* segment playing on) blended with the incoming head as the render's picture blends them, and the rest of the
|
|
640
641
|
* incoming segment follows.
|
|
641
642
|
*/
|
|
642
|
-
function buildDrawnJoin(input: { joined: string; incoming: string; output: string;
|
|
643
|
-
const { joined, incoming, output, drawn, index } = input;
|
|
644
|
-
const at = Math.round(input.plan.start * FOOTAGE_FPS);
|
|
643
|
+
function buildDrawnJoin(input: { joined: string; incoming: string; output: string; at: number; drawn: { kind: DrawnTransition; frames: number }; frame: Frame; index: number }): string[] {
|
|
644
|
+
const { joined, incoming, output, drawn, index, at } = input;
|
|
645
645
|
const p = `d${String(index)}`;
|
|
646
646
|
return [
|
|
647
647
|
`[${joined}]split[${p}a][${p}b]`,
|
|
@@ -656,17 +656,31 @@ function buildDrawnJoin(input: { joined: string; incoming: string; output: strin
|
|
|
656
656
|
];
|
|
657
657
|
}
|
|
658
658
|
|
|
659
|
+
/** The draft's picture graph over a run of segments: its inputs, its filters, and the label of the joined stream. */
|
|
660
|
+
interface DraftPictureGraph {
|
|
661
|
+
inputs: string[];
|
|
662
|
+
parts: string[];
|
|
663
|
+
joined: string;
|
|
664
|
+
inputCount: number;
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
interface DraftPictureGraphInput {
|
|
668
|
+
input: FootageDraftArgsInput;
|
|
669
|
+
plans: readonly DraftSegmentPlan[];
|
|
670
|
+
draft: Frame;
|
|
671
|
+
threads: { global: string[]; output: string[] };
|
|
672
|
+
/** The first and last segment the graph plays; the joined stream starts at the first segment's start. */
|
|
673
|
+
first: number;
|
|
674
|
+
last: number;
|
|
675
|
+
}
|
|
676
|
+
|
|
659
677
|
/**
|
|
660
|
-
*
|
|
661
|
-
*
|
|
662
|
-
*
|
|
663
|
-
* exactly the cut's length.
|
|
678
|
+
* Segments `first` to `last` read from their clips, zoomed and joined as the draft joins them (`xfade` at a fade,
|
|
679
|
+
* `concat` at the others, a drawn transition blended in), on a stream that starts at segment `first`'s start. The
|
|
680
|
+
* whole cut (`first` 0) gives exactly the one-run draft's graph.
|
|
664
681
|
*/
|
|
665
|
-
|
|
682
|
+
function buildDraftPictureGraph({ input, plans, draft, threads, first, last }: DraftPictureGraphInput): DraftPictureGraph {
|
|
666
683
|
const { edl } = input;
|
|
667
|
-
const draft = getFootageDraftFrame(input.frame, input.width);
|
|
668
|
-
const plans = planFootageDraft(input.film, edl, input.overlays ?? []);
|
|
669
|
-
const threads = getThreadArgs(input.threads, "draft.threads");
|
|
670
684
|
const inputs: string[] = [];
|
|
671
685
|
const parts: string[] = [];
|
|
672
686
|
let inputCount = 0;
|
|
@@ -675,7 +689,9 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
675
689
|
if (clip === undefined) throw new Error(`No file for clip "${id}" of the EDL.`);
|
|
676
690
|
return clip;
|
|
677
691
|
};
|
|
678
|
-
|
|
692
|
+
const base = (plans[first] as DraftSegmentPlan).start;
|
|
693
|
+
for (let index = first; index <= last; index += 1) {
|
|
694
|
+
const segment = edl.segments[index] as Edl["segments"][number];
|
|
679
695
|
const plan = plans[index] as DraftSegmentPlan;
|
|
680
696
|
const fit = [
|
|
681
697
|
"setsar=1",
|
|
@@ -694,65 +710,78 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
694
710
|
// A split screen holds still: its three bands and its layer, at the draft's size, as the render's picture draws them.
|
|
695
711
|
const layer = input.splitLayers?.[segment.id];
|
|
696
712
|
if (layer === undefined) throw new Error(`No split layer for segment "${segment.id}" of the EDL.`);
|
|
697
|
-
const
|
|
713
|
+
const bandsFrom = inputCount;
|
|
698
714
|
for (const band of split.bands) inputs.push(...threads.output, "-ss", String(r3(band.start)), "-t", String(plan.seconds), "-i", getClip(band.clip));
|
|
699
715
|
inputs.push(...getSplitLayerInputArgs(layer, plan.seconds, FOOTAGE_FPS));
|
|
700
716
|
inputCount += 4;
|
|
701
|
-
const bands: [string, string, string] = [`${String(
|
|
702
|
-
parts.push(...buildSplitScreenFilters({ inputs: bands, layer: `${String(
|
|
703
|
-
|
|
717
|
+
const bands: [string, string, string] = [`${String(bandsFrom)}:v`, `${String(bandsFrom + 1)}:v`, `${String(bandsFrom + 2)}:v`];
|
|
718
|
+
parts.push(...buildSplitScreenFilters({ inputs: bands, layer: `${String(bandsFrom + 3)}:v`, split, frame: draft, fit: ["setpts=PTS-STARTPTS", ...fit], output: `v${index}`, prefix: `sp${String(index)}` }));
|
|
719
|
+
continue;
|
|
704
720
|
}
|
|
705
721
|
// Before an input, -threads sizes that input's decoder: one decoder runs per segment.
|
|
706
722
|
inputs.push(...threads.output, "-ss", String(r3(segment.start)), "-t", String(plan.seconds), "-i", getClip(segment.clip));
|
|
707
723
|
parts.push(`[${inputCount}:v]${["setpts=PTS-STARTPTS", ...buildPictureFilters(plan, draft), ...fit].join(",")}[v${index}]`);
|
|
708
724
|
inputCount += 1;
|
|
709
|
-
}
|
|
710
|
-
let joined =
|
|
711
|
-
|
|
712
|
-
|
|
725
|
+
}
|
|
726
|
+
let joined = `v${String(first)}`;
|
|
727
|
+
for (let index = first + 1; index <= last; index += 1) {
|
|
728
|
+
const plan = plans[index] as DraftSegmentPlan;
|
|
713
729
|
const label = `j${index}`;
|
|
730
|
+
// The joined stream starts at segment `first`: offsets are counted from there (from 0 for the whole cut).
|
|
731
|
+
const start = base === 0 ? plan.start : r3(plan.start - base);
|
|
714
732
|
if (plan.drawn !== null) {
|
|
715
|
-
|
|
733
|
+
const at = base === 0 ? Math.round(plan.start * FOOTAGE_FPS) : Math.round(plan.start * FOOTAGE_FPS) - Math.round(base * FOOTAGE_FPS);
|
|
734
|
+
parts.push(...buildDrawnJoin({ joined, incoming: `v${index}`, output: label, at, drawn: plan.drawn, frame: draft, index }));
|
|
716
735
|
joined = label;
|
|
717
|
-
|
|
736
|
+
continue;
|
|
718
737
|
}
|
|
719
738
|
parts.push(
|
|
720
739
|
plan.fade > 0
|
|
721
|
-
? `[${joined}][v${index}]xfade=transition=fade:duration=${plan.fade}:offset=${
|
|
740
|
+
? `[${joined}][v${index}]xfade=transition=fade:duration=${plan.fade}:offset=${start}[${label}]`
|
|
722
741
|
: // concat changes the time base; xfade needs both of its inputs on the segments' own.
|
|
723
742
|
`[${joined}][v${index}]concat=n=2:v=1:a=0,settb=1/${FOOTAGE_FPS},fps=${FOOTAGE_FPS}[${label}]`,
|
|
724
743
|
);
|
|
725
744
|
joined = label;
|
|
726
|
-
});
|
|
727
|
-
if (input.captions !== null) {
|
|
728
|
-
parts.push(`[${joined}]ass=filename=${input.captions}${input.fontsDir === null ? "" : `:fontsdir=${input.fontsDir}`}[draft]`);
|
|
729
|
-
} else {
|
|
730
|
-
parts.push(`[${joined}]null[draft]`);
|
|
731
745
|
}
|
|
732
|
-
|
|
746
|
+
return { inputs, parts, joined, inputCount };
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
/** The draft's sound inputs and filters, and its encoder's arguments: shared by the one-run draft and the join. */
|
|
750
|
+
interface DraftSound {
|
|
751
|
+
inputs: string[];
|
|
752
|
+
parts: string[];
|
|
753
|
+
map: string;
|
|
754
|
+
codec: string[];
|
|
755
|
+
}
|
|
756
|
+
|
|
757
|
+
/** The draft's sound read from input `soundInput` on: the film's prepared mix copied, or the speech (and music) mixed. */
|
|
758
|
+
function getDraftSound(input: Pick<FootageDraftArgsInput, "speech" | "music" | "mix">, soundInput: number): DraftSound {
|
|
733
759
|
const mix = input.mix ?? null;
|
|
734
760
|
const music = mix === null ? (input.music ?? null) : null;
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
761
|
+
return {
|
|
762
|
+
inputs: ["-i", mix ?? input.speech, ...(music === null ? [] : ["-i", music])],
|
|
763
|
+
parts: music === null ? [] : [`[${soundInput}:a][${soundInput + 1}:a]amix=inputs=2:duration=first:normalize=0[sound]`],
|
|
764
|
+
map: music === null ? `${soundInput}:a` : "[sound]",
|
|
765
|
+
// A prepared mix is copied: it was measured as encoded, and a second encode would move its loudness and true peak.
|
|
766
|
+
codec: mix === null ? ["-c:a", "aac", "-b:a", DRAFT_AUDIO_BITRATE] : ["-c:a", "copy"],
|
|
767
|
+
};
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
/** The filter that burns the captions into `joined` (or passes it through) as `[draft]`. */
|
|
771
|
+
function getCaptionsFilter(joined: string, input: Pick<FootageDraftArgsInput, "captions" | "fontsDir">): string {
|
|
772
|
+
if (input.captions === null) return `[${joined}]null[draft]`;
|
|
773
|
+
return `[${joined}]ass=filename=${input.captions}${input.fontsDir === null ? "" : `:fontsdir=${input.fontsDir}`}[draft]`;
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
/** The draft's output: the picture and sound mapped, exactly `seconds` long, H.264 yuv420p with fast start. */
|
|
777
|
+
function getDraftOutputArgs(input: { sound: DraftSound; seconds: number; threads: string[]; output: string }): string[] {
|
|
739
778
|
return [
|
|
740
|
-
"-v",
|
|
741
|
-
"error",
|
|
742
|
-
"-y",
|
|
743
|
-
...threads.global,
|
|
744
|
-
...inputs,
|
|
745
|
-
"-i",
|
|
746
|
-
mix ?? input.speech,
|
|
747
|
-
...(music === null ? [] : ["-i", music]),
|
|
748
|
-
"-filter_complex",
|
|
749
|
-
parts.join(";"),
|
|
750
779
|
"-map",
|
|
751
780
|
"[draft]",
|
|
752
781
|
"-map",
|
|
753
|
-
|
|
782
|
+
input.sound.map,
|
|
754
783
|
"-t",
|
|
755
|
-
String(seconds),
|
|
784
|
+
String(input.seconds),
|
|
756
785
|
"-c:v",
|
|
757
786
|
"libx264",
|
|
758
787
|
"-preset",
|
|
@@ -761,14 +790,196 @@ export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
|
761
790
|
String(DRAFT_CRF),
|
|
762
791
|
"-pix_fmt",
|
|
763
792
|
"yuv420p",
|
|
764
|
-
...
|
|
793
|
+
...input.sound.codec,
|
|
765
794
|
"-movflags",
|
|
766
795
|
"+faststart",
|
|
796
|
+
...input.threads,
|
|
797
|
+
input.output,
|
|
798
|
+
];
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/**
|
|
802
|
+
* The ffmpeg arguments of the draft in one run (`assembly: "single"`): each segment read from its own seek into its
|
|
803
|
+
* clip, zoomed and scaled to the draft's width, joined by `xfade` at `fade` cuts and `concat` at the others, the
|
|
804
|
+
* captions burned in (`ass`), the film's mix copied under it (or the speech and music tracks mixed and encoded to AAC);
|
|
805
|
+
* H.264 yuv420p with fast start, exactly the cut's length. Every segment keeps its own decoder for the whole run, so
|
|
806
|
+
* its memory grows with the number of cuts (`estimateFootageDraftMemory`).
|
|
807
|
+
*/
|
|
808
|
+
export function buildFootageDraftArgs(input: FootageDraftArgsInput): string[] {
|
|
809
|
+
const { edl } = input;
|
|
810
|
+
const draft = getFootageDraftFrame(input.frame, input.width);
|
|
811
|
+
const plans = planFootageDraft(input.film, edl, input.overlays ?? []);
|
|
812
|
+
const threads = getThreadArgs(input.threads, "draft.threads");
|
|
813
|
+
const graph = buildDraftPictureGraph({ input, plans, draft, threads, first: 0, last: edl.segments.length - 1 });
|
|
814
|
+
const sound = getDraftSound(input, graph.inputCount);
|
|
815
|
+
const parts = [...graph.parts, getCaptionsFilter(graph.joined, input), ...sound.parts];
|
|
816
|
+
return ["-v", "error", "-y", ...threads.global, ...graph.inputs, ...sound.inputs, "-filter_complex", parts.join(";"), ...getDraftOutputArgs({ sound, seconds: getEdlDuration(edl), threads: threads.output, output: input.output })];
|
|
817
|
+
}
|
|
818
|
+
|
|
819
|
+
/** How a footage draft's picture is assembled: in pieces then joined (bounded memory), or in one ffmpeg run. */
|
|
820
|
+
export const FOOTAGE_DRAFT_ASSEMBLIES = ["pieces", "single"] as const;
|
|
821
|
+
export type FootageDraftAssembly = (typeof FOOTAGE_DRAFT_ASSEMBLIES)[number];
|
|
822
|
+
|
|
823
|
+
/** How many segments a piece of a `pieces` draft holds by default: one ffmpeg run decodes at most these (and a lead). */
|
|
824
|
+
export const FOOTAGE_DRAFT_PIECE_SEGMENTS = 4;
|
|
825
|
+
/** The CRF of a piece: near transparent, so the join's one encode at the draft's CRF sets the draft's quality. */
|
|
826
|
+
const DRAFT_PIECE_CRF = 16;
|
|
827
|
+
|
|
828
|
+
/** One piece of a `pieces` draft: a run of whole segments, cut out of a graph that may start one segment earlier. */
|
|
829
|
+
export interface DraftPiece {
|
|
830
|
+
/** The piece's first and last segment. */
|
|
831
|
+
first: number;
|
|
832
|
+
last: number;
|
|
833
|
+
/**
|
|
834
|
+
* The segment the piece's graph starts from: `first - 1` when segment `first` starts on a fade or a drawn transition
|
|
835
|
+
* (its outgoing tail plays under it, as in the one-run draft), `first` otherwise.
|
|
836
|
+
*/
|
|
837
|
+
from: number;
|
|
838
|
+
/** The frames of the graph's stream before segment `first`: the lead segment's own, trimmed away. */
|
|
839
|
+
skipFrames: number;
|
|
840
|
+
/** The piece's first film frame and its frame count: the pieces cover the cut's frames without a gap. */
|
|
841
|
+
frame: number;
|
|
842
|
+
frames: number;
|
|
843
|
+
}
|
|
844
|
+
|
|
845
|
+
/** The film frame a film second falls on. */
|
|
846
|
+
function toDraftFrame(seconds: number): number {
|
|
847
|
+
return Math.round(seconds * FOOTAGE_FPS);
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
/**
|
|
851
|
+
* The pieces of a `pieces` draft: `segmentsPerPiece` segments each (a positive whole number), the last piece what is
|
|
852
|
+
* left. Throws on an EDL without segments.
|
|
853
|
+
*/
|
|
854
|
+
export function planFootageDraftPieces(plans: readonly DraftSegmentPlan[], seconds: number, segmentsPerPiece: number = FOOTAGE_DRAFT_PIECE_SEGMENTS): DraftPiece[] {
|
|
855
|
+
if (!Number.isInteger(segmentsPerPiece) || segmentsPerPiece < 1) throw new Error(`draft.segmentsPerPiece must be a positive whole number, got ${segmentsPerPiece}.`);
|
|
856
|
+
if (plans.length === 0) throw new Error("The EDL has no segments.");
|
|
857
|
+
const pieces: DraftPiece[] = [];
|
|
858
|
+
for (let first = 0; first < plans.length; first += segmentsPerPiece) {
|
|
859
|
+
const last = Math.min(plans.length, first + segmentsPerPiece) - 1;
|
|
860
|
+
const plan = plans[first] as DraftSegmentPlan;
|
|
861
|
+
const from = first > 0 && (plan.fade > 0 || plan.drawn !== null) ? first - 1 : first;
|
|
862
|
+
const frame = toDraftFrame(plan.start);
|
|
863
|
+
const next = plans[last + 1];
|
|
864
|
+
const end = next === undefined ? toDraftFrame(seconds) : toDraftFrame(next.start);
|
|
865
|
+
pieces.push({ first, last, from, skipFrames: frame - toDraftFrame((plans[from] as DraftSegmentPlan).start), frame, frames: Math.max(1, end - frame) });
|
|
866
|
+
}
|
|
867
|
+
return pieces;
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
export interface FootageDraftPieceArgsInput extends Omit<FootageDraftArgsInput, "speech" | "music" | "mix" | "captions" | "fontsDir"> {
|
|
871
|
+
piece: DraftPiece;
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
/**
|
|
875
|
+
* One piece's ffmpeg arguments: its segments (and a lead segment's tail) as the one-run draft draws them, cut to the
|
|
876
|
+
* piece's own frames, H.264 without sound or captions. A piece decodes only its own segments, so its memory does not
|
|
877
|
+
* grow with the cut.
|
|
878
|
+
*/
|
|
879
|
+
export function buildFootageDraftPieceArgs(input: FootageDraftPieceArgsInput): string[] {
|
|
880
|
+
const { piece } = input;
|
|
881
|
+
const draft = getFootageDraftFrame(input.frame, input.width);
|
|
882
|
+
const plans = planFootageDraft(input.film, input.edl, input.overlays ?? []);
|
|
883
|
+
const threads = getThreadArgs(input.threads, "draft.threads");
|
|
884
|
+
const full = { ...input, speech: "", captions: null, fontsDir: null };
|
|
885
|
+
const graph = buildDraftPictureGraph({ input: full, plans, draft, threads, first: piece.from, last: piece.last });
|
|
886
|
+
const cut = `[${graph.joined}]trim=start_frame=${String(piece.skipFrames)}:end_frame=${String(piece.skipFrames + piece.frames)},setpts=PTS-STARTPTS[picture]`;
|
|
887
|
+
return [
|
|
888
|
+
"-v",
|
|
889
|
+
"error",
|
|
890
|
+
"-y",
|
|
891
|
+
...threads.global,
|
|
892
|
+
...graph.inputs,
|
|
893
|
+
"-filter_complex",
|
|
894
|
+
[...graph.parts, cut].join(";"),
|
|
895
|
+
"-map",
|
|
896
|
+
"[picture]",
|
|
897
|
+
"-an",
|
|
898
|
+
"-frames:v",
|
|
899
|
+
String(piece.frames),
|
|
900
|
+
"-c:v",
|
|
901
|
+
"libx264",
|
|
902
|
+
"-preset",
|
|
903
|
+
"veryfast",
|
|
904
|
+
"-crf",
|
|
905
|
+
String(DRAFT_PIECE_CRF),
|
|
906
|
+
"-pix_fmt",
|
|
907
|
+
"yuv420p",
|
|
767
908
|
...threads.output,
|
|
768
909
|
input.output,
|
|
769
910
|
];
|
|
770
911
|
}
|
|
771
912
|
|
|
913
|
+
export interface FootageDraftJoinArgsInput extends Pick<FootageDraftArgsInput, "speech" | "music" | "mix" | "captions" | "fontsDir" | "threads" | "output"> {
|
|
914
|
+
/** The concat demuxer's list of the pieces (`buildPictureList`). */
|
|
915
|
+
list: string;
|
|
916
|
+
/** The cut's length. */
|
|
917
|
+
seconds: number;
|
|
918
|
+
}
|
|
919
|
+
|
|
920
|
+
/**
|
|
921
|
+
* The join of a `pieces` draft: the pieces read one after the other by the concat demuxer (one decoder), the captions
|
|
922
|
+
* burned in once over the whole film, the sound as the one-run draft takes it, the same encoder and length.
|
|
923
|
+
*/
|
|
924
|
+
export function buildFootageDraftJoinArgs(input: FootageDraftJoinArgsInput): string[] {
|
|
925
|
+
const threads = getThreadArgs(input.threads, "draft.threads");
|
|
926
|
+
const sound = getDraftSound(input, 1);
|
|
927
|
+
const parts = [getCaptionsFilter("0:v", input), ...sound.parts];
|
|
928
|
+
return ["-v", "error", "-y", ...threads.global, ...threads.output, "-f", "concat", "-safe", "0", "-i", input.list, ...sound.inputs, "-filter_complex", parts.join(";"), ...getDraftOutputArgs({ sound, seconds: input.seconds, threads: threads.output, output: input.output })];
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
/** The working memory of a draft's ffmpeg run that does not grow with the cut (the encoder, libass, the graph's base). */
|
|
932
|
+
const DRAFT_BASE_MB = 250;
|
|
933
|
+
/** One decoder of a 1080x1920 source with its zoom and frame queues, as measured on a production worker (about 70 MB). */
|
|
934
|
+
const DRAFT_DECODER_MB_AT_1080P = 70;
|
|
935
|
+
const FULL_HD_PIXELS = 1080 * 1920;
|
|
936
|
+
|
|
937
|
+
export interface FootageDraftMemoryInput {
|
|
938
|
+
/** The cut's segments; only a split screen's bands count (each opens its own decoder). */
|
|
939
|
+
edl: { segments: readonly { split?: { bands: readonly unknown[] } | undefined }[] };
|
|
940
|
+
/** The normalized clips' frame (the film's): a decoder's memory grows with its pixels. */
|
|
941
|
+
frame: Frame;
|
|
942
|
+
assembly: FootageDraftAssembly;
|
|
943
|
+
/** The segments of a piece (`pieces`); default `FOOTAGE_DRAFT_PIECE_SEGMENTS`. */
|
|
944
|
+
segmentsPerPiece?: number;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
/** What a draft's largest ffmpeg run decodes at once, and the working memory that takes by the kit's estimate. */
|
|
948
|
+
export interface FootageDraftMemory {
|
|
949
|
+
/** The decoders of the largest run: every segment's (three for a split screen) in `single`, a piece's in `pieces`. */
|
|
950
|
+
decoders: number;
|
|
951
|
+
/** The estimated peak working memory of that run, in MB (`250 + decoders x 70 x pixels / 1080x1920`). */
|
|
952
|
+
peakMb: number;
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
/** The decoders a segment opens: its clip, or a split screen's three bands. */
|
|
956
|
+
function countDecoders(segment: FootageDraftMemoryInput["edl"]["segments"][number]): number {
|
|
957
|
+
return segment.split === undefined ? 1 : segment.split.bands.length;
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
/**
|
|
961
|
+
* The draft's peak memory by the kit's estimate, before it runs: a `single` draft opens every segment's decoder at
|
|
962
|
+
* once, a `pieces` draft at most one piece's (with a lead segment). An app compares it with its worker's memory limit
|
|
963
|
+
* and warns before a draft that would not fit.
|
|
964
|
+
*/
|
|
965
|
+
export function estimateFootageDraftMemory(input: FootageDraftMemoryInput): FootageDraftMemory {
|
|
966
|
+
const segments = input.edl.segments;
|
|
967
|
+
const per = segments.map(countDecoders);
|
|
968
|
+
let decoders = per.reduce((sum, count) => sum + count, 0);
|
|
969
|
+
if (input.assembly === "pieces") {
|
|
970
|
+
const size = input.segmentsPerPiece ?? FOOTAGE_DRAFT_PIECE_SEGMENTS;
|
|
971
|
+
if (!Number.isInteger(size) || size < 1) throw new Error(`draft.segmentsPerPiece must be a positive whole number, got ${size}.`);
|
|
972
|
+
decoders = 0;
|
|
973
|
+
for (let first = 0; first < per.length; first += size) {
|
|
974
|
+
// A lead segment is counted whenever there is one before: the estimate does not read transitions.
|
|
975
|
+
const piece = per.slice(Math.max(0, first - 1), first + size).reduce((sum, count) => sum + count, 0);
|
|
976
|
+
decoders = Math.max(decoders, piece);
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
const decoderMb = (DRAFT_DECODER_MB_AT_1080P * input.frame.width * input.frame.height) / FULL_HD_PIXELS;
|
|
980
|
+
return { decoders, peakMb: Math.round(DRAFT_BASE_MB + decoders * decoderMb) };
|
|
981
|
+
}
|
|
982
|
+
|
|
772
983
|
export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buildDir" | "log" | "signal" | "onProgress" | "onFfmpegFailure">, Partial<Pick<FootageRunOptions, "cacheDir" | "providers">> {
|
|
773
984
|
/** The draft MP4. */
|
|
774
985
|
output: string;
|
|
@@ -787,6 +998,15 @@ export interface RenderFootageDraftOptions extends Pick<FootageRunOptions, "buil
|
|
|
787
998
|
* own choice, which follows the host's cores. Set it from a CPU-limited container's quota.
|
|
788
999
|
*/
|
|
789
1000
|
threads?: number;
|
|
1001
|
+
/**
|
|
1002
|
+
* How the picture is assembled. `pieces` (the default): `segmentsPerPiece` segments per ffmpeg run, written to
|
|
1003
|
+
* `<buildDir>/draft/pieces/` and joined with the captions burned in once, so the memory is one piece's whatever the
|
|
1004
|
+
* cut's length; slower. `single`: one ffmpeg run over the whole cut (`buildFootageDraftArgs`), faster, its memory
|
|
1005
|
+
* growing with the number of cuts (`estimateFootageDraftMemory`).
|
|
1006
|
+
*/
|
|
1007
|
+
assembly?: FootageDraftAssembly;
|
|
1008
|
+
/** The segments of a piece in `pieces`; default `FOOTAGE_DRAFT_PIECE_SEGMENTS`. A positive whole number. */
|
|
1009
|
+
segmentsPerPiece?: number;
|
|
790
1010
|
}
|
|
791
1011
|
|
|
792
1012
|
/** The draft's music: the film has none, it played (a file, or generated music from the cache), or it is not cached yet. */
|
|
@@ -799,6 +1019,9 @@ export interface FootageDraft {
|
|
|
799
1019
|
captionFont: string | null;
|
|
800
1020
|
/** The draft's length: the cut's. */
|
|
801
1021
|
seconds: number;
|
|
1022
|
+
/** How the picture was assembled, and in how many pieces (1 for `single`). */
|
|
1023
|
+
assembly: FootageDraftAssembly;
|
|
1024
|
+
pieces: number;
|
|
802
1025
|
width: number;
|
|
803
1026
|
height: number;
|
|
804
1027
|
bytes: number;
|
|
@@ -839,8 +1062,9 @@ async function findDraftMusic(film: FootageFilm, edl: Edl, clips: PreparedClip[]
|
|
|
839
1062
|
|
|
840
1063
|
/**
|
|
841
1064
|
* Makes the draft of a cut film: the speech track, the ducked music track (when it costs nothing), the captions file
|
|
842
|
-
* and its font in `<buildDir>/draft/`, then
|
|
843
|
-
*
|
|
1065
|
+
* and its font in `<buildDir>/draft/`, then the picture: in pieces joined with the captions (`assembly: "pieces"`, the
|
|
1066
|
+
* default, memory bounded by one piece) or in one ffmpeg run (`"single"`, `buildFootageDraftArgs`). Spends nothing: no
|
|
1067
|
+
* provider is called; takes the same clips, transcript and EDL as `renderFootage`.
|
|
844
1068
|
*/
|
|
845
1069
|
export function renderFootageDraft(film: FootageFilm, clips: PreparedClip[], transcript: Extract<TranscribeOutcome, { kind: "ready" }>, edl: Edl, options: RenderFootageDraftOptions): Promise<FootageResult<FootageDraft>> {
|
|
846
1070
|
return withFfmpegFailureReporting(getFfmpegReporting(options), () => runRenderFootageDraft(film, clips, transcript, edl, options));
|
|
@@ -969,24 +1193,25 @@ async function runRenderFootageDraft(
|
|
|
969
1193
|
}
|
|
970
1194
|
report("film", 0);
|
|
971
1195
|
const splitLayers = writeSplitLayers(edl.segments, getFootageDraftFrame(frame, options.width), draftDir, true);
|
|
1196
|
+
const assembly = options.assembly ?? "pieces";
|
|
972
1197
|
// Run in the draft folder: the captions file and the font folder are relative names, so no path needs escaping in the filter graph.
|
|
973
|
-
const
|
|
1198
|
+
const picture = {
|
|
974
1199
|
film,
|
|
975
1200
|
edl,
|
|
976
1201
|
clips: clipFiles,
|
|
977
|
-
speech: speechFile,
|
|
978
|
-
music: musicFile,
|
|
979
|
-
mix: prepared?.audioFile ?? null,
|
|
980
1202
|
frame,
|
|
981
1203
|
...(options.width === undefined ? {} : { width: options.width }),
|
|
982
1204
|
...(options.threads === undefined ? {} : { threads: options.threads }),
|
|
983
|
-
captions: captions === null ? null : "captions.ass",
|
|
984
1205
|
...(overlays.length === 0 ? {} : { overlays }),
|
|
985
|
-
fontsDir: captions === null ? null : "fonts",
|
|
986
1206
|
...(Object.keys(splitLayers).length === 0 ? {} : { splitLayers }),
|
|
987
|
-
|
|
988
|
-
}
|
|
989
|
-
|
|
1207
|
+
};
|
|
1208
|
+
const sound = { speech: speechFile, music: musicFile, mix: prepared?.audioFile ?? null, captions: captions === null ? null : "captions.ass", fontsDir: captions === null ? null : "fonts" };
|
|
1209
|
+
let pieceCount = 1;
|
|
1210
|
+
if (assembly === "single") {
|
|
1211
|
+
await runFfmpeg("The footage draft", buildFootageDraftArgs({ ...picture, ...sound, output }), { ...aborting, output, cwd: draftDir, onSeconds: (done) => report("film", done) });
|
|
1212
|
+
} else {
|
|
1213
|
+
pieceCount = await assembleDraftInPieces({ picture, sound, seconds, draftDir, output, aborting, segmentsPerPiece: options.segmentsPerPiece, onSeconds: (done) => report("film", done) });
|
|
1214
|
+
}
|
|
990
1215
|
const measured = await measureDraftLoudness(output, mixLoudness, prepared !== null, aborting);
|
|
991
1216
|
if (!measured.ok) return measured;
|
|
992
1217
|
const draftLoudness = measured.value;
|
|
@@ -1005,6 +1230,8 @@ async function runRenderFootageDraft(
|
|
|
1005
1230
|
music: draftMusic,
|
|
1006
1231
|
captionFont: captions === null ? null : font.family,
|
|
1007
1232
|
seconds,
|
|
1233
|
+
assembly,
|
|
1234
|
+
pieces: pieceCount,
|
|
1008
1235
|
width: draft.width,
|
|
1009
1236
|
height: draft.height,
|
|
1010
1237
|
bytes,
|
|
@@ -1014,6 +1241,57 @@ async function runRenderFootageDraft(
|
|
|
1014
1241
|
});
|
|
1015
1242
|
}
|
|
1016
1243
|
|
|
1244
|
+
interface PiecesRun {
|
|
1245
|
+
picture: Omit<FootageDraftPieceArgsInput, "piece" | "output">;
|
|
1246
|
+
sound: Pick<FootageDraftArgsInput, "speech" | "music" | "mix" | "captions" | "fontsDir">;
|
|
1247
|
+
seconds: number;
|
|
1248
|
+
draftDir: string;
|
|
1249
|
+
output: string;
|
|
1250
|
+
aborting: { signal?: AbortSignal };
|
|
1251
|
+
segmentsPerPiece: number | undefined;
|
|
1252
|
+
/** Seconds of the film done: the pieces take the first 80 %, the join the rest. */
|
|
1253
|
+
onSeconds: (seconds: number) => void;
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
/** The share of the film step the pieces take; the join takes the rest. */
|
|
1257
|
+
const PIECES_SHARE = 0.8;
|
|
1258
|
+
|
|
1259
|
+
/**
|
|
1260
|
+
* Draws the picture piece by piece into `<draftDir>/pieces/` (one ffmpeg at a time), then joins the pieces with the
|
|
1261
|
+
* captions and the sound into `output`. The pieces are removed after the join; a failed run leaves them for its
|
|
1262
|
+
* snapshot. Answers the number of pieces.
|
|
1263
|
+
*/
|
|
1264
|
+
async function assembleDraftInPieces(run: PiecesRun): Promise<number> {
|
|
1265
|
+
const { picture, aborting, draftDir } = run;
|
|
1266
|
+
const plans = planFootageDraft(picture.film, picture.edl, picture.overlays ?? []);
|
|
1267
|
+
const pieces = planFootageDraftPieces(plans, run.seconds, run.segmentsPerPiece);
|
|
1268
|
+
const piecesDir = join(draftDir, "pieces");
|
|
1269
|
+
mkdirSync(piecesDir, { recursive: true });
|
|
1270
|
+
const files: string[] = [];
|
|
1271
|
+
for (const [index, piece] of pieces.entries()) {
|
|
1272
|
+
const name = `piece-${String(index).padStart(4, "0")}.mp4`;
|
|
1273
|
+
const file = join(piecesDir, name);
|
|
1274
|
+
const at = piece.frame / FOOTAGE_FPS;
|
|
1275
|
+
await runFfmpeg(`The footage draft's piece ${String(index + 1)} of ${String(pieces.length)}`, buildFootageDraftPieceArgs({ ...picture, piece, output: join("pieces", name) }), {
|
|
1276
|
+
...aborting,
|
|
1277
|
+
output: file,
|
|
1278
|
+
cwd: draftDir,
|
|
1279
|
+
onSeconds: (done) => run.onSeconds(Math.min(run.seconds, at + done) * PIECES_SHARE),
|
|
1280
|
+
});
|
|
1281
|
+
files.push(name);
|
|
1282
|
+
}
|
|
1283
|
+
const list = join(piecesDir, "pieces.txt");
|
|
1284
|
+
writeFileSync(list, buildPictureList(files));
|
|
1285
|
+
await runFfmpeg("The footage draft's join", buildFootageDraftJoinArgs({ ...run.sound, list: join("pieces", "pieces.txt"), seconds: run.seconds, ...(picture.threads === undefined ? {} : { threads: picture.threads }), output: run.output }), {
|
|
1286
|
+
...aborting,
|
|
1287
|
+
output: run.output,
|
|
1288
|
+
cwd: draftDir,
|
|
1289
|
+
onSeconds: (done) => run.onSeconds(run.seconds * PIECES_SHARE + Math.min(run.seconds, done) * (1 - PIECES_SHARE)),
|
|
1290
|
+
});
|
|
1291
|
+
rmSync(piecesDir, { recursive: true, force: true });
|
|
1292
|
+
return pieces.length;
|
|
1293
|
+
}
|
|
1294
|
+
|
|
1017
1295
|
/**
|
|
1018
1296
|
* The written draft's sound measured as `finishFootage` measures the finished film: held to `mixLoudness` when the mix
|
|
1019
1297
|
* stage made it (a miss fails, a near miss inside `warnTolerance` carries `warning`), reported otherwise; null when
|
|
@@ -35,6 +35,11 @@ import {
|
|
|
35
35
|
getPlaceCardPinSize,
|
|
36
36
|
getListCardsByItem,
|
|
37
37
|
getListItemTimes,
|
|
38
|
+
LIST_CHIP_BADGE_EM,
|
|
39
|
+
LIST_CHIP_COVERED_OPACITY,
|
|
40
|
+
getChipCoveredTimes,
|
|
41
|
+
getChipParts,
|
|
42
|
+
getChipTickPoints,
|
|
38
43
|
getNoteColors,
|
|
39
44
|
getNotePaperPoints,
|
|
40
45
|
getNoteTextBox,
|
|
@@ -116,7 +121,12 @@ function getOverlayCss(colors: OverlayColors, fonts: { heading: string; interfac
|
|
|
116
121
|
`.ov.center .ov-card{transform-origin:50% 100%}`,
|
|
117
122
|
// A centred keyword (style.align center) centres a line that wraps too.
|
|
118
123
|
`.ov-keyword.center .ov-card{justify-content:center;text-align:center}`,
|
|
119
|
-
`.ov-list .ov-chip{position:absolute;box-sizing:border-box;flex-direction:row;align-items:center;justify-content:
|
|
124
|
+
`.ov-list .ov-chip{position:absolute;box-sizing:border-box;flex-direction:row;align-items:center;justify-content:flex-start;gap:${LIST_CHIP_BADGE_EM.gap}em;white-space:nowrap;line-height:${OVERLAY_LINE_HEIGHT};transform-origin:50% 100%}`,
|
|
125
|
+
// A chip's step badge (RV-181 of an adopting app): a round accent mark with its number, a tick once the next point is in.
|
|
126
|
+
`.ov-chip span{flex:1;text-align:center}`,
|
|
127
|
+
`.ov-num{position:relative;flex:none;display:grid;place-items:center;border-radius:50%;background:${colors.accent};color:${colors.background};font-style:normal;font-weight:800;line-height:1}`,
|
|
128
|
+
`.ov-num i{font-style:normal;font-size:.8em}`,
|
|
129
|
+
`.ov-num svg{position:absolute;inset:0;width:100%;height:100%;overflow:visible}`,
|
|
120
130
|
`.ov-svg{position:absolute;inset:0;width:100%;height:100%;overflow:visible}`,
|
|
121
131
|
`.ov-search,.ov-counter .ov-card,.ov-date .ov-card,.ov-place,.ov-sub,.ov-line{white-space:nowrap}`,
|
|
122
132
|
// A title card fills its fitted box, so a typed title keeps its width; a centred typed line hides its untyped
|
|
@@ -402,15 +412,32 @@ function addList(overlay: Extract<ResolvedOverlay, { kind: "list" }>, id: string
|
|
|
402
412
|
const wipe = (suffix: string) => (overlay.entrance === "wipe" ? `<i class="ov-wipe" id="${id}w${suffix}"></i>` : "");
|
|
403
413
|
const style = `left:${box.left}px;top:${box.top}px;width:${box.width}px;height:${box.height}px;font-size:${fontSize}px;align-items:flex-start`;
|
|
404
414
|
if (overlay.chips !== undefined) {
|
|
415
|
+
const covered = getChipCoveredTimes(overlay);
|
|
405
416
|
const chips = overlay.chips.map((chip, i) => {
|
|
406
|
-
|
|
407
|
-
|
|
417
|
+
// The badge sits as far from the chip's left edge as from its top.
|
|
418
|
+
const place = `left:${chip.left - box.left}px;top:${chip.top - box.top}px;${getListCardSize(chip, pad)} ${pad.y}px ${pad.y}px${i === 0 ? "" : ";visibility:hidden"}${own === "" ? "" : `;${own}`}`;
|
|
419
|
+
const { radius } = getChipParts(chip, overlay).badge;
|
|
420
|
+
const size = 2 * radius;
|
|
421
|
+
const local = getChipTickPoints({ centre: { x: radius, y: radius }, radius });
|
|
422
|
+
const tick =
|
|
423
|
+
covered[i] === null
|
|
424
|
+
? ""
|
|
425
|
+
: `<svg viewBox="0 0 ${size} ${size}"><polyline id="${id}k${i}" points="${local.map((point) => `${point.x},${point.y}`).join(" ")}" fill="none" stroke="${colors.background}" stroke-width="${r3(radius * 0.24)}" stroke-linecap="round" stroke-linejoin="round" pathLength="1" stroke-dasharray="1" stroke-dashoffset="1"/></svg>`;
|
|
426
|
+
const badge = `<b class="ov-num" style="width:${size}px;height:${size}px"><i id="${id}n${i}">${i + 1}</i>${tick}</b>`;
|
|
427
|
+
return `<div class="ov-card ov-chip" id="${id}c${i}" style="${place}">${wipe(String(i))}${badge}<span id="${id}t${i}">${escapeHtml(overlay.items[i] as string)}</span></div>`;
|
|
408
428
|
});
|
|
409
429
|
out.html.push(clipDiv(overlay, id, "ov ov-list chips", style, chips.join("")));
|
|
410
430
|
times.forEach((at, i) => {
|
|
411
431
|
if (i > 0) out.tweens.push(`tl.set("#${id}c${i}", { visibility: "visible" }, ${at});`);
|
|
412
432
|
out.tweens.push(...getListCardEntrance(overlay, `#${id}c${i}`, at, context));
|
|
413
433
|
});
|
|
434
|
+
// Each point leads until the next one is in, then steps back: fainter, its number turned into a tick (RV-181).
|
|
435
|
+
covered.forEach((at, i) => {
|
|
436
|
+
if (at === null) return;
|
|
437
|
+
out.tweens.push(`tl.to("#${id}t${i}", { opacity: ${LIST_CHIP_COVERED_OPACITY}, duration: 0.3, ease: "power2.out" }, ${at});`);
|
|
438
|
+
out.tweens.push(`tl.to("#${id}n${i}", { opacity: 0, duration: 0.12, ease: "none" }, ${at});`);
|
|
439
|
+
out.tweens.push(`tl.fromTo("#${id}k${i}", { attr: { "stroke-dashoffset": 1 } }, { attr: { "stroke-dashoffset": 0 }, duration: 0.3, ease: "power2.out", immediateRender: false }, ${r3(at + 0.08)});`);
|
|
440
|
+
});
|
|
414
441
|
} else {
|
|
415
442
|
const isCentred = overlay.isCentred === true;
|
|
416
443
|
const dotStyle = overlay.colors.accent === undefined ? "" : ` style="background:${colors.accent}"`;
|
|
@@ -84,8 +84,8 @@ export type OverlayPin = (typeof OVERLAY_PINS)[number];
|
|
|
84
84
|
|
|
85
85
|
/**
|
|
86
86
|
* How a list draws its points: `stack` (one centred card, its points one under another, the card growing as each one
|
|
87
|
-
* comes in), `chips` (each point its own small card, side by side in one
|
|
88
|
-
* or `auto` (chips when every point fits one row at the list's size, else the stack). Without it: `stack`.
|
|
87
|
+
* comes in), `chips` (each point its own small card, side by side in one row on the frame's axis, numbered, entering
|
|
88
|
+
* one by one as a countdown: the newest point leads, the earlier ones step back with a tick) or `auto` (chips when every point fits one row at the list's size, else the stack). Without it: `stack`.
|
|
89
89
|
*/
|
|
90
90
|
export const OVERLAY_LIST_LAYOUTS = ["stack", "chips", "auto"] as const;
|
|
91
91
|
|
|
@@ -483,7 +483,7 @@ export const overlaySchema = z
|
|
|
483
483
|
style: z
|
|
484
484
|
.strictObject({
|
|
485
485
|
size: z.enum(OVERLAY_SIZES).optional().describe("s, m (the default) or l: the punch-in's zoom (108, 112, 115 %) or the type size."),
|
|
486
|
-
side: z.enum(OVERLAY_SIDES).optional().describe("The side a keyword, lower third, list, date stamp or note sits on."),
|
|
486
|
+
side: z.enum(OVERLAY_SIDES).optional().describe("The side a keyword, lower third, list, date stamp or note sits on; a note without a side sits on the frame's axis."),
|
|
487
487
|
position: z
|
|
488
488
|
.enum(OVERLAY_POSITIONS)
|
|
489
489
|
.optional()
|
|
@@ -526,7 +526,7 @@ export const overlaySchema = z
|
|
|
526
526
|
.enum(OVERLAY_LIST_LAYOUTS)
|
|
527
527
|
.optional()
|
|
528
528
|
.describe(
|
|
529
|
-
"A list only: stack (the default: one centred card growing as each point comes in), chips (each point its own small card in one centred row,
|
|
529
|
+
"A list only: stack (the default: one centred card growing as each point comes in), chips (each point its own small numbered card in one centred row, coming in one by one as a countdown: the newest leads, the earlier ones step back with a tick) or auto (chips when they fit one row, else stack).",
|
|
530
530
|
),
|
|
531
531
|
subtitle_delay: z
|
|
532
532
|
.number()
|