@softure-ai/marketing-kit 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/README.md +108 -3
- package/dist/cli/footage.d.ts +6 -0
- package/dist/cli/footage.d.ts.map +1 -0
- package/dist/cli/footage.js +95 -0
- package/dist/cli/footage.js.map +1 -0
- package/dist/cli/main.js +18 -1
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/options.d.ts +18 -3
- package/dist/cli/options.d.ts.map +1 -1
- package/dist/cli/options.js +41 -2
- package/dist/cli/options.js.map +1 -1
- package/dist/compose/compose.d.ts.map +1 -1
- package/dist/compose/compose.js +1 -22
- package/dist/compose/compose.js.map +1 -1
- package/dist/compose/html.d.ts +6 -0
- package/dist/compose/html.d.ts.map +1 -0
- package/dist/compose/html.js +24 -0
- package/dist/compose/html.js.map +1 -0
- package/dist/config/actions-schema.d.ts +14 -14
- package/dist/config/config.d.ts +16 -0
- package/dist/config/config.d.ts.map +1 -1
- package/dist/config/config.js +88 -0
- package/dist/config/config.js.map +1 -1
- package/dist/config/schema.d.ts +133 -70
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +110 -3
- package/dist/config/schema.js.map +1 -1
- package/dist/config/shot-steps.d.ts +16 -16
- package/dist/footage/cache.d.ts +23 -0
- package/dist/footage/cache.d.ts.map +1 -0
- package/dist/footage/cache.js +51 -0
- package/dist/footage/cache.js.map +1 -0
- package/dist/footage/compose.d.ts +50 -0
- package/dist/footage/compose.d.ts.map +1 -0
- package/dist/footage/compose.js +166 -0
- package/dist/footage/compose.js.map +1 -0
- package/dist/footage/edl.d.ts +83 -0
- package/dist/footage/edl.d.ts.map +1 -0
- package/dist/footage/edl.js +166 -0
- package/dist/footage/edl.js.map +1 -0
- package/dist/footage/elevenlabs.d.ts +31 -0
- package/dist/footage/elevenlabs.d.ts.map +1 -0
- package/dist/footage/elevenlabs.js +140 -0
- package/dist/footage/elevenlabs.js.map +1 -0
- package/dist/footage/fake.d.ts +38 -0
- package/dist/footage/fake.d.ts.map +1 -0
- package/dist/footage/fake.js +66 -0
- package/dist/footage/fake.js.map +1 -0
- package/dist/footage/film.d.ts +103 -0
- package/dist/footage/film.d.ts.map +1 -0
- package/dist/footage/film.js +29 -0
- package/dist/footage/film.js.map +1 -0
- package/dist/footage/music.d.ts +37 -0
- package/dist/footage/music.d.ts.map +1 -0
- package/dist/footage/music.js +63 -0
- package/dist/footage/music.js.map +1 -0
- package/dist/footage/normalize.d.ts +41 -0
- package/dist/footage/normalize.d.ts.map +1 -0
- package/dist/footage/normalize.js +114 -0
- package/dist/footage/normalize.js.map +1 -0
- package/dist/footage/pipeline.d.ts +105 -0
- package/dist/footage/pipeline.d.ts.map +1 -0
- package/dist/footage/pipeline.js +385 -0
- package/dist/footage/pipeline.js.map +1 -0
- package/dist/footage/probe.d.ts +64 -0
- package/dist/footage/probe.d.ts.map +1 -0
- package/dist/footage/probe.js +138 -0
- package/dist/footage/probe.js.map +1 -0
- package/dist/footage/providers.d.ts +61 -0
- package/dist/footage/providers.d.ts.map +1 -0
- package/dist/footage/providers.js +13 -0
- package/dist/footage/providers.js.map +1 -0
- package/dist/footage/render.d.ts +20 -0
- package/dist/footage/render.d.ts.map +1 -0
- package/dist/footage/render.js +76 -0
- package/dist/footage/render.js.map +1 -0
- package/dist/index.d.ts +13 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -2
- package/dist/index.js.map +1 -1
- package/dist/render/render.d.ts +6 -0
- package/dist/render/render.d.ts.map +1 -1
- package/dist/render/render.js +1 -1
- package/dist/render/render.js.map +1 -1
- package/package.json +1 -1
- package/schema/marketing.schema.json +254 -2
- package/src/cli/footage.ts +87 -0
- package/src/cli/main.ts +17 -1
- package/src/cli/options.ts +51 -4
- package/src/compose/compose.ts +1 -25
- package/src/compose/html.ts +28 -0
- package/src/config/config.ts +97 -0
- package/src/config/schema.ts +114 -4
- package/src/footage/cache.ts +66 -0
- package/src/footage/compose.ts +213 -0
- package/src/footage/edl.ts +204 -0
- package/src/footage/elevenlabs.ts +154 -0
- package/src/footage/fake.ts +92 -0
- package/src/footage/film.ts +111 -0
- package/src/footage/music.ts +86 -0
- package/src/footage/normalize.ts +133 -0
- package/src/footage/pipeline.ts +447 -0
- package/src/footage/probe.ts +172 -0
- package/src/footage/providers.ts +71 -0
- package/src/footage/render.ts +82 -0
- package/src/index.ts +109 -1
- package/src/render/render.ts +1 -1
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { getTrimmedDuration } from "./probe.js";
|
|
3
|
+
/**
|
|
4
|
+
* One intermediate format for every clip, so the composition can cut between them freely: the film's frame at
|
|
5
|
+
* 30 fps, upright (ffmpeg applies the stored rotation while decoding), SDR BT.709 (HDR tone-mapped), filled by a crop
|
|
6
|
+
* or a blurred copy, a keyframe every half second (the renderer seeks into each segment), and 48 kHz stereo AAC
|
|
7
|
+
* (silence when the clip has no sound), so the speech track can be cut from any clip alike.
|
|
8
|
+
*/
|
|
9
|
+
export const FOOTAGE_FPS = 30;
|
|
10
|
+
export const FOOTAGE_SAMPLE_RATE = 48_000;
|
|
11
|
+
/** Within this difference of aspect ratios `auto` crops; beyond it, it fills with a blurred copy. */
|
|
12
|
+
const CROP_ASPECT_TOLERANCE = 0.15;
|
|
13
|
+
/** The fit `auto` resolves to for this picture in this frame. */
|
|
14
|
+
export function resolveFit(fit, probe, frame) {
|
|
15
|
+
if (fit !== "auto")
|
|
16
|
+
return fit;
|
|
17
|
+
const clipAspect = probe.displayWidth / probe.displayHeight;
|
|
18
|
+
const frameAspect = frame.width / frame.height;
|
|
19
|
+
return Math.abs(clipAspect - frameAspect) / frameAspect <= CROP_ASPECT_TOLERANCE ? "crop" : "blur";
|
|
20
|
+
}
|
|
21
|
+
const TRANSFER_INPUT = { pq: "smpte2084", hlg: "arib-std-b67", "dolby-vision": "arib-std-b67" };
|
|
22
|
+
/**
|
|
23
|
+
* HDR to SDR BT.709 on the CPU: to linear light (zscale), Hable tone curve, back to BT.709. A Dolby Vision phone
|
|
24
|
+
* clip is read through its HLG base layer.
|
|
25
|
+
*/
|
|
26
|
+
export function toneMapFilter(hdr, transfer) {
|
|
27
|
+
if (hdr === null)
|
|
28
|
+
return [];
|
|
29
|
+
const input = transfer === "smpte2084" || transfer === "arib-std-b67" ? transfer : TRANSFER_INPUT[hdr];
|
|
30
|
+
return [
|
|
31
|
+
`zscale=tin=${input}:pin=bt2020:min=bt2020nc:t=linear:npl=100`,
|
|
32
|
+
"format=gbrpf32le",
|
|
33
|
+
"zscale=p=bt709",
|
|
34
|
+
"tonemap=tonemap=hable:desat=0",
|
|
35
|
+
"zscale=t=bt709:m=bt709:r=tv",
|
|
36
|
+
// Back to 8-bit before scaling: the blur fill on float frames crashed ffmpeg 6.1.
|
|
37
|
+
"format=yuv420p",
|
|
38
|
+
];
|
|
39
|
+
}
|
|
40
|
+
/** The video filter graph from input 0's picture to `[v]`. */
|
|
41
|
+
export function buildVideoFilter(options) {
|
|
42
|
+
const { width, height } = options.frame;
|
|
43
|
+
const head = [...toneMapFilter(options.hdr, options.transfer), `fps=${FOOTAGE_FPS}`].join(",");
|
|
44
|
+
const cover = `scale=${width}:${height}:force_original_aspect_ratio=increase,crop=${width}:${height}`;
|
|
45
|
+
const tail = "setsar=1,format=yuv420p";
|
|
46
|
+
if (options.fit === "crop")
|
|
47
|
+
return `[0:v]${head},${cover},${tail}[v]`;
|
|
48
|
+
return [
|
|
49
|
+
`[0:v]${head},split=2[bg][fg]`,
|
|
50
|
+
`[bg]scale=${Math.round(width / 4)}:${Math.round(height / 4)}:force_original_aspect_ratio=increase,crop=${Math.round(width / 4)}:${Math.round(height / 4)},boxblur=20:2,scale=${width}:${height},eq=brightness=-0.06[back]`,
|
|
51
|
+
`[fg]scale=${width}:${height}:force_original_aspect_ratio=decrease[front]`,
|
|
52
|
+
`[back][front]overlay=(W-w)/2:(H-h)/2,${tail}[v]`,
|
|
53
|
+
].join(";");
|
|
54
|
+
}
|
|
55
|
+
/** ffmpeg's arguments for one clip; pure, so the graph is tested without ffmpeg. */
|
|
56
|
+
export function buildNormalizeArgs(input) {
|
|
57
|
+
const { clip, file, probe, frame, output } = input;
|
|
58
|
+
const duration = getTrimmedDuration(clip, probe);
|
|
59
|
+
const fit = resolveFit(clip.fit, probe, frame);
|
|
60
|
+
const seek = clip.trim.start > 0 ? ["-ss", String(clip.trim.start)] : [];
|
|
61
|
+
const silence = probe.hasAudio ? [] : ["-f", "lavfi", "-i", `anullsrc=r=${FOOTAGE_SAMPLE_RATE}:cl=stereo`];
|
|
62
|
+
return [
|
|
63
|
+
"-v",
|
|
64
|
+
"error",
|
|
65
|
+
"-y",
|
|
66
|
+
...seek,
|
|
67
|
+
"-i",
|
|
68
|
+
file,
|
|
69
|
+
...silence,
|
|
70
|
+
"-t",
|
|
71
|
+
duration.toFixed(3),
|
|
72
|
+
"-filter_complex",
|
|
73
|
+
buildVideoFilter({ fit, frame, hdr: probe.hdr, transfer: probe.colorTransfer }),
|
|
74
|
+
"-map",
|
|
75
|
+
"[v]",
|
|
76
|
+
"-map",
|
|
77
|
+
probe.hasAudio ? "0:a:0" : "1:a",
|
|
78
|
+
"-c:v",
|
|
79
|
+
"libx264",
|
|
80
|
+
"-preset",
|
|
81
|
+
"veryfast",
|
|
82
|
+
"-crf",
|
|
83
|
+
"18",
|
|
84
|
+
"-g",
|
|
85
|
+
String(FOOTAGE_FPS / 2),
|
|
86
|
+
"-color_primaries",
|
|
87
|
+
"bt709",
|
|
88
|
+
"-color_trc",
|
|
89
|
+
"bt709",
|
|
90
|
+
"-colorspace",
|
|
91
|
+
"bt709",
|
|
92
|
+
"-c:a",
|
|
93
|
+
"aac",
|
|
94
|
+
"-b:a",
|
|
95
|
+
"192k",
|
|
96
|
+
"-ar",
|
|
97
|
+
String(FOOTAGE_SAMPLE_RATE),
|
|
98
|
+
"-ac",
|
|
99
|
+
"2",
|
|
100
|
+
"-movflags",
|
|
101
|
+
"+faststart",
|
|
102
|
+
output,
|
|
103
|
+
];
|
|
104
|
+
}
|
|
105
|
+
/** Runs ffmpeg for one clip; throws with the clip's id when it fails (a broken file is not an expected outcome here). */
|
|
106
|
+
export function normalizeFootage(input) {
|
|
107
|
+
const result = spawnSync("ffmpeg", buildNormalizeArgs(input), { stdio: ["ignore", "inherit", "inherit"] });
|
|
108
|
+
if (result.error !== undefined)
|
|
109
|
+
throw new Error(`Normalizing clip ${input.clip.id}: ffmpeg did not start (${result.error.message}).`);
|
|
110
|
+
if (result.status !== 0) {
|
|
111
|
+
throw new Error(`Normalizing clip ${input.clip.id} (${input.file}): ffmpeg ended with code ${String(result.status)} (signal ${result.signal ?? "-"}).`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
//# sourceMappingURL=normalize.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"normalize.js","sourceRoot":"","sources":["../../src/footage/normalize.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAG/C,OAAO,EAAE,kBAAkB,EAAmC,MAAM,YAAY,CAAC;AAEjF;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,WAAW,GAAG,EAAE,CAAC;AAC9B,MAAM,CAAC,MAAM,mBAAmB,GAAG,MAAM,CAAC;AAC1C,qGAAqG;AACrG,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAOnC,iEAAiE;AACjE,MAAM,UAAU,UAAU,CAAC,GAAY,EAAE,KAA2D,EAAE,KAAY;IAChH,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,GAAG,CAAC;IAC/B,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,aAAa,CAAC;IAC5D,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC/C,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,WAAW,CAAC,GAAG,WAAW,IAAI,qBAAqB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AACrG,CAAC;AAED,MAAM,cAAc,GAA4B,EAAE,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,CAAC;AAEzH;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,GAAmB,EAAE,QAAuB;IACxE,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,QAAQ,KAAK,WAAW,IAAI,QAAQ,KAAK,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;IACvG,OAAO;QACL,cAAc,KAAK,2CAA2C;QAC9D,kBAAkB;QAClB,gBAAgB;QAChB,+BAA+B;QAC/B,6BAA6B;QAC7B,kFAAkF;QAClF,gBAAgB;KACjB,CAAC;AACJ,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,gBAAgB,CAAC,OAA6F;IAC5H,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;IACxC,MAAM,IAAI,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,OAAO,WAAW,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC/F,MAAM,KAAK,GAAG,SAAS,KAAK,IAAI,MAAM,8CAA8C,KAAK,IAAI,MAAM,EAAE,CAAC;IACtG,MAAM,IAAI,GAAG,yBAAyB,CAAC;IACvC,IAAI,OAAO,CAAC,GAAG,KAAK,MAAM;QAAE,OAAO,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC;IACtE,OAAO;QACL,QAAQ,IAAI,kBAAkB;QAC9B,aAAa,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,8CAA8C,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,uBAAuB,KAAK,IAAI,MAAM,4BAA4B;QAC3N,aAAa,KAAK,IAAI,MAAM,8CAA8C;QAC1E,wCAAwC,IAAI,KAAK;KAClD,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACd,CAAC;AAWD,oFAAoF;AACpF,MAAM,UAAU,kBAAkB,CAAC,KAAqB;IACtD,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IACnD,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACjD,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACzE,MAAM,OAAO,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,mBAAmB,YAAY,CAAC,CAAC;IAC3G,OAAO;QACL,IAAI;QACJ,OAAO;QACP,IAAI;QACJ,GAAG,IAAI;QACP,IAAI;QACJ,IAAI;QACJ,GAAG,OAAO;QACV,IAAI;QACJ,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QACnB,iBAAiB;QACjB,gBAAgB,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,aAAa,EAAE,CAAC;QAC/E,MAAM;QACN,KAAK;QACL,MAAM;QACN,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;QAChC,MAAM;QACN,SAAS;QACT,SAAS;QACT,UAAU;QACV,MAAM;QACN,IAAI;QACJ,IAAI;QACJ,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC;QACvB,kBAAkB;QAClB,OAAO;QACP,YAAY;QACZ,OAAO;QACP,aAAa;QACb,OAAO;QACP,MAAM;QACN,KAAK;QACL,MAAM;QACN,MAAM;QACN,KAAK;QACL,MAAM,CAAC,mBAAmB,CAAC;QAC3B,KAAK;QACL,GAAG;QACH,WAAW;QACX,YAAY;QACZ,MAAM;KACP,CAAC;AACJ,CAAC;AAED,yHAAyH;AACzH,MAAM,UAAU,gBAAgB,CAAC,KAAqB;IACpD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,EAAE,kBAAkB,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IAC3G,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,KAAK,CAAC,IAAI,CAAC,EAAE,2BAA2B,MAAM,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;IACtI,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,oBAAoB,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,6BAA6B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,YAAY,MAAM,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC;IAC1J,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { type LayoutOverride } from "../compose/timeline.js";
|
|
2
|
+
import type { BrandColors } from "../config/colors.js";
|
|
3
|
+
import type { BrandFont } from "../config/config.js";
|
|
4
|
+
import type { SfxEvent } from "../config/schema.js";
|
|
5
|
+
import { type TtsResult } from "../voice/provider.js";
|
|
6
|
+
import type { TimedWord } from "../voice/voiceover.js";
|
|
7
|
+
import { type Edl } from "./edl.js";
|
|
8
|
+
import type { FootageClip, FootageFilm } from "./film.js";
|
|
9
|
+
import { type Frame } from "./normalize.js";
|
|
10
|
+
import { type FootageProbe } from "./probe.js";
|
|
11
|
+
import { type FootageProviders, type PaidStep } from "./providers.js";
|
|
12
|
+
export interface FootageBrand {
|
|
13
|
+
name: string;
|
|
14
|
+
/** BCP 47, e.g. `en-US`. */
|
|
15
|
+
locale: string;
|
|
16
|
+
colors: BrandColors;
|
|
17
|
+
fonts: {
|
|
18
|
+
heading: BrandFont | null;
|
|
19
|
+
body: BrandFont | null;
|
|
20
|
+
};
|
|
21
|
+
logo: string | null;
|
|
22
|
+
}
|
|
23
|
+
export interface FootageRunOptions {
|
|
24
|
+
/** The film's build folder (`<buildDir>/<film id>` in the CLI). */
|
|
25
|
+
buildDir: string;
|
|
26
|
+
/** The paid cache's root; files go to `<cacheDir>/<film id>/`. */
|
|
27
|
+
cacheDir: string;
|
|
28
|
+
providers: FootageProviders;
|
|
29
|
+
/** Only `true` lets a provider spend money. */
|
|
30
|
+
isCommit: boolean;
|
|
31
|
+
isPlaceholder: boolean;
|
|
32
|
+
log: (line: string) => void;
|
|
33
|
+
/** Downloads clip URLs; injected in tests. */
|
|
34
|
+
fetch?: typeof globalThis.fetch;
|
|
35
|
+
}
|
|
36
|
+
export interface PreparedClip {
|
|
37
|
+
clip: FootageClip;
|
|
38
|
+
/** The source as read (a URL's downloaded copy). */
|
|
39
|
+
source: string;
|
|
40
|
+
probe: FootageProbe;
|
|
41
|
+
/** The normalized clip. */
|
|
42
|
+
file: string;
|
|
43
|
+
/** Seconds after the trim. */
|
|
44
|
+
duration: number;
|
|
45
|
+
}
|
|
46
|
+
export type FootageResult<T> = TtsResult<T>;
|
|
47
|
+
export type TranscribeOutcome = {
|
|
48
|
+
kind: "dry-run";
|
|
49
|
+
steps: PaidStep[];
|
|
50
|
+
} | {
|
|
51
|
+
kind: "ready";
|
|
52
|
+
/** Each clip's words, times from its trimmed start. */
|
|
53
|
+
words: Record<string, TimedWord[]>;
|
|
54
|
+
/** The audio each clip's speech is cut from (the cleaned voice when isolated). */
|
|
55
|
+
speech: Record<string, string>;
|
|
56
|
+
};
|
|
57
|
+
export declare function getFrame(film: Pick<FootageFilm, "format">): Frame;
|
|
58
|
+
export declare function getEdlPath(buildDir: string, isPlaceholder: boolean): string;
|
|
59
|
+
/** Downloads URL clips, probes every clip (all problems at once), normalizes each one that changed. */
|
|
60
|
+
export declare function prepareFootage(film: FootageFilm, options: Pick<FootageRunOptions, "buildDir" | "log" | "fetch">): Promise<FootageResult<PreparedClip[]>>;
|
|
61
|
+
/** The paid music step, when the film's music still has to be bought; null when it is cached or free. */
|
|
62
|
+
export declare function getMusicPaidStep(film: FootageFilm, options: Pick<FootageRunOptions, "cacheDir" | "providers">, seconds: number): PaidStep | null;
|
|
63
|
+
/**
|
|
64
|
+
* Each talking clip's words with times, from the cache or (with `isCommit`) from the speech-to-text provider, the
|
|
65
|
+
* voice cleaned first when the film isolates it. Without `isCommit`, a cache miss returns the dry run: every paid
|
|
66
|
+
* call the film still needs, music included, with its length.
|
|
67
|
+
*/
|
|
68
|
+
export declare function transcribeFootage(film: FootageFilm, clips: PreparedClip[], options: FootageRunOptions): Promise<FootageResult<TranscribeOutcome>>;
|
|
69
|
+
/** The cut from the transcripts, written to the EDL file (replacing what is there). */
|
|
70
|
+
export declare function cutFootage(film: FootageFilm, clips: PreparedClip[], words: Record<string, TimedWord[]>, edlPath: string): Edl;
|
|
71
|
+
/** The EDL file as written by `cutFootage` or edited by hand, checked against the clips. */
|
|
72
|
+
export declare function readEdlFile(film: FootageFilm, clips: PreparedClip[], edlPath: string): FootageResult<Edl>;
|
|
73
|
+
export interface RenderFootageOptions extends FootageRunOptions {
|
|
74
|
+
brand: FootageBrand;
|
|
75
|
+
sfx: Partial<Record<SfxEvent, string>>;
|
|
76
|
+
/** hyperframes' `--quality`: `draft`, `standard` or `high`. */
|
|
77
|
+
quality: string;
|
|
78
|
+
/** The finished MP4. */
|
|
79
|
+
output: string;
|
|
80
|
+
/** A project's override of the format's caption box (`marketing.json` `layout`). */
|
|
81
|
+
layout?: LayoutOverride;
|
|
82
|
+
}
|
|
83
|
+
export type RenderedFootage = {
|
|
84
|
+
kind: "dry-run";
|
|
85
|
+
steps: PaidStep[];
|
|
86
|
+
} | {
|
|
87
|
+
kind: "rendered";
|
|
88
|
+
output: string;
|
|
89
|
+
seconds: number;
|
|
90
|
+
edlPath: string;
|
|
91
|
+
};
|
|
92
|
+
/** Composes the cut and renders it: speech track, ducked music, end card backdrop, assets, hyperframes. */
|
|
93
|
+
export declare function renderFootage(film: FootageFilm, clips: PreparedClip[], transcript: Extract<TranscribeOutcome, {
|
|
94
|
+
kind: "ready";
|
|
95
|
+
}>, edl: Edl, options: RenderFootageOptions): Promise<FootageResult<RenderedFootage>>;
|
|
96
|
+
export interface RenderFootageFilmOptions extends RenderFootageOptions {
|
|
97
|
+
/** Build the EDL again even when the build folder holds one (by default a present EDL, maybe edited by hand, is used). */
|
|
98
|
+
isRecut?: boolean;
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* The whole footage film from a `FootageFilm` object, no `marketing.json` needed: prepare, transcribe (or the dry
|
|
102
|
+
* run), cut (reusing an EDL already in the build folder unless `isRecut`), render.
|
|
103
|
+
*/
|
|
104
|
+
export declare function renderFootageFilm(film: FootageFilm, options: RenderFootageFilmOptions): Promise<FootageResult<RenderedFootage>>;
|
|
105
|
+
//# sourceMappingURL=pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/footage/pipeline.ts"],"names":[],"mappings":"AAKA,OAAO,EAA0B,KAAK,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACrF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAGpD,OAAO,EAAW,KAAK,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAC/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAGvD,OAAO,EAAkD,KAAK,GAAG,EAAE,MAAM,UAAU,CAAC;AAEpF,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE1D,OAAO,EAAoB,KAAK,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAC9D,OAAO,EAAmD,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAChG,OAAO,EAAqB,KAAK,gBAAgB,EAAE,KAAK,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAkBzF,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,4BAA4B;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,WAAW,CAAC;IACpB,KAAK,EAAE;QAAE,OAAO,EAAE,SAAS,GAAG,IAAI,CAAC;QAAC,IAAI,EAAE,SAAS,GAAG,IAAI,CAAA;KAAE,CAAC;IAC7D,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,kEAAkE;IAClE,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,gBAAgB,CAAC;IAC5B,+CAA+C;IAC/C,QAAQ,EAAE,OAAO,CAAC;IAClB,aAAa,EAAE,OAAO,CAAC;IACvB,GAAG,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5B,8CAA8C;IAC9C,KAAK,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACjC;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,WAAW,CAAC;IAClB,oDAAoD;IACpD,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,YAAY,CAAC;IACpB,2BAA2B;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,8BAA8B;IAC9B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,aAAa,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;AAE5C,MAAM,MAAM,iBAAiB,GACzB;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE,QAAQ,EAAE,CAAA;CAAE,GACtC;IACE,IAAI,EAAE,OAAO,CAAC;IACd,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;IACnC,kFAAkF;IAClF,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAChC,CAAC;AAEN,wBAAgB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE,QAAQ,CAAC,GAAG,KAAK,CAEjE;AAED,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,GAAG,MAAM,CAE3E;AA8BD,uGAAuG;AACvG,wBAAsB,cAAc,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,EAAE,UAAU,GAAG,KAAK,GAAG,OAAO,CAAC,GAAG,OAAO,CAAC,aAAa,CAAC,YAAY,EAAE,CAAC,CAAC,CAgC9J;AA0BD,yGAAyG;AACzG,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,CAAC,iBAAiB,EAAE,UAAU,GAAG,WAAW,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAKhJ;AASD;;;;GAIG;AACH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,CAkEvJ;AAED,uFAAuF;AACvF,wBAAgB,UAAU,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,MAAM,GAAG,GAAG,CAU7H;AAED,4FAA4F;AAC5F,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,CAWzG;AA2CD,MAAM,WAAW,oBAAqB,SAAQ,iBAAiB;IAC7D,KAAK,EAAE,YAAY,CAAC;IACpB,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IACvC,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,oFAAoF;IACpF,MAAM,CAAC,EAAE,cAAc,CAAC;CACzB;AAED,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,KAAK,EAAE,QAAQ,EAAE,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,UAAU,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAE9I,2GAA2G;AAC3G,wBAAsB,aAAa,CACjC,IAAI,EAAE,WAAW,EACjB,KAAK,EAAE,YAAY,EAAE,EACrB,UAAU,EAAE,OAAO,CAAC,iBAAiB,EAAE;IAAE,IAAI,EAAE,OAAO,CAAA;CAAE,CAAC,EACzD,GAAG,EAAE,GAAG,EACR,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CA6DzC;AAED,MAAM,WAAW,wBAAyB,SAAQ,oBAAoB;IACpE,0HAA0H;IAC1H,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,wBAAwB,GAAG,OAAO,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAkBrI"}
|
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { createRequire } from "node:module";
|
|
4
|
+
import { dirname, extname, join, resolve } from "node:path";
|
|
5
|
+
import { LAYOUTS, resolveLayout } from "../compose/timeline.js";
|
|
6
|
+
import { runHyperframes } from "../render/hyperframes.js";
|
|
7
|
+
import { copyFonts } from "../render/render.js";
|
|
8
|
+
import { err, ok } from "../voice/provider.js";
|
|
9
|
+
import { footageCacheKey, getFootageCachePath, hashFile, readCachedTranscript, writeCacheFile, writeTranscript } from "./cache.js";
|
|
10
|
+
import { composeFootage, END_CARD_SECONDS, getFootageFilmSeconds } from "./compose.js";
|
|
11
|
+
import { buildEdl, findEdlProblems, readEdl, remapWords } from "./edl.js";
|
|
12
|
+
import { makePlaceholderTranscript } from "./fake.js";
|
|
13
|
+
import { duckingKeyframes } from "./music.js";
|
|
14
|
+
import { normalizeFootage } from "./normalize.js";
|
|
15
|
+
import { describeProbe, getTrimmedDuration, probeFootage } from "./probe.js";
|
|
16
|
+
import { describePaidSteps } from "./providers.js";
|
|
17
|
+
import { buildAudioArgs, buildCutPreviewArgs, buildLastFrameArgs, buildMusicArgs, buildSpeechArgs, runFfmpeg } from "./render.js";
|
|
18
|
+
/**
|
|
19
|
+
* The footage film end to end, as standalone steps (`prepareFootage` → `transcribeFootage` → `cutFootage` →
|
|
20
|
+
* `renderFootage`) and as one call (`renderFootageFilm`). Paid calls happen only with `isCommit`; without it a step
|
|
21
|
+
* that would pay returns a dry run with the estimate. `isPlaceholder` replaces every paid call with a free stand-in
|
|
22
|
+
* (a transcript from each clip's `script`, no isolation, no generated music) and keeps its files apart from the real
|
|
23
|
+
* film's (`placeholder/` in the build folder, its own EDL).
|
|
24
|
+
*
|
|
25
|
+
* Build folder layout: `sources/` (downloaded URLs), `clips/<id>.mp4` (normalized), `audio/<id>.mp3`, `edl.json`
|
|
26
|
+
* (`edl.placeholder.json`), `assets/`, `index.html`.
|
|
27
|
+
*/
|
|
28
|
+
const require = createRequire(import.meta.url);
|
|
29
|
+
/** Bumped when the normalized format changes, so older normalized clips are made again. */
|
|
30
|
+
const NORMALIZE_VERSION = 1;
|
|
31
|
+
export function getFrame(film) {
|
|
32
|
+
return LAYOUTS[film.format].frame;
|
|
33
|
+
}
|
|
34
|
+
export function getEdlPath(buildDir, isPlaceholder) {
|
|
35
|
+
return join(buildDir, isPlaceholder ? "edl.placeholder.json" : "edl.json");
|
|
36
|
+
}
|
|
37
|
+
const isUrl = (source) => /^https?:\/\//i.test(source);
|
|
38
|
+
async function download(url, dir, fetch, log) {
|
|
39
|
+
const name = createHash("sha256").update(url).digest("hex").slice(0, 16);
|
|
40
|
+
const extension = extname(new URL(url).pathname).toLowerCase() || ".mp4";
|
|
41
|
+
const file = join(dir, `${name}${extension}`);
|
|
42
|
+
if (existsSync(file))
|
|
43
|
+
return ok(file);
|
|
44
|
+
log(`download: ${url}`);
|
|
45
|
+
let response;
|
|
46
|
+
try {
|
|
47
|
+
response = await fetch(url);
|
|
48
|
+
}
|
|
49
|
+
catch (error) {
|
|
50
|
+
return err(`downloading ${url} failed: ${String(error)}`);
|
|
51
|
+
}
|
|
52
|
+
if (!response.ok)
|
|
53
|
+
return err(`downloading ${url}: the server answered ${response.status}`);
|
|
54
|
+
mkdirSync(dir, { recursive: true });
|
|
55
|
+
writeFileSync(`${file}.part`, Buffer.from(await response.arrayBuffer()));
|
|
56
|
+
cpSync(`${file}.part`, file);
|
|
57
|
+
rmSync(`${file}.part`, { force: true });
|
|
58
|
+
return ok(file);
|
|
59
|
+
}
|
|
60
|
+
/** The stamp that says a normalized clip was made from this source with these settings. */
|
|
61
|
+
function getNormalizeStamp(clip, source, frame) {
|
|
62
|
+
const stat = statSync(source);
|
|
63
|
+
return JSON.stringify({ version: NORMALIZE_VERSION, source: resolve(source), size: stat.size, mtime: stat.mtimeMs, trim: clip.trim, fit: clip.fit, frame });
|
|
64
|
+
}
|
|
65
|
+
/** Downloads URL clips, probes every clip (all problems at once), normalizes each one that changed. */
|
|
66
|
+
export async function prepareFootage(film, options) {
|
|
67
|
+
const { buildDir, log } = options;
|
|
68
|
+
if (film.clips.length === 0)
|
|
69
|
+
return err(`footage film ${film.id} has no clips.`);
|
|
70
|
+
const sources = {};
|
|
71
|
+
for (const clip of film.clips) {
|
|
72
|
+
if (!isUrl(clip.source)) {
|
|
73
|
+
if (!existsSync(clip.source))
|
|
74
|
+
return err(`clip "${clip.id}": no file at ${clip.source}`);
|
|
75
|
+
sources[clip.id] = clip.source;
|
|
76
|
+
continue;
|
|
77
|
+
}
|
|
78
|
+
const downloaded = await download(clip.source, join(buildDir, "sources"), options.fetch ?? globalThis.fetch, log);
|
|
79
|
+
if (!downloaded.ok)
|
|
80
|
+
return err(`clip "${clip.id}": ${downloaded.error}`);
|
|
81
|
+
sources[clip.id] = downloaded.value;
|
|
82
|
+
}
|
|
83
|
+
const probed = probeFootage(film.clips, sources);
|
|
84
|
+
if (!probed.ok)
|
|
85
|
+
return err(probed.errors.join("\n"));
|
|
86
|
+
const frame = getFrame(film);
|
|
87
|
+
const clipsDir = join(buildDir, "clips");
|
|
88
|
+
mkdirSync(clipsDir, { recursive: true });
|
|
89
|
+
const prepared = [];
|
|
90
|
+
for (const { clip, file: source, probe } of probed.clips) {
|
|
91
|
+
log(describeProbe(clip, probe));
|
|
92
|
+
const file = join(clipsDir, `${clip.id}.mp4`);
|
|
93
|
+
const stampPath = join(clipsDir, `${clip.id}.json`);
|
|
94
|
+
const stamp = getNormalizeStamp(clip, source, frame);
|
|
95
|
+
if (!existsSync(file) || !existsSync(stampPath) || readFileSync(stampPath, "utf8") !== stamp) {
|
|
96
|
+
normalizeFootage({ clip, file: source, probe, frame, output: file });
|
|
97
|
+
writeFileSync(stampPath, stamp);
|
|
98
|
+
}
|
|
99
|
+
prepared.push({ clip, source, probe, file, duration: getTrimmedDuration(clip, probe) });
|
|
100
|
+
}
|
|
101
|
+
return ok(prepared);
|
|
102
|
+
}
|
|
103
|
+
/** The cache identity of a clip's source part: its bytes and its trim. */
|
|
104
|
+
async function getClipKey(clip) {
|
|
105
|
+
return footageCacheKey("transcript", { source: await hashFile(clip.source), trim: clip.clip.trim });
|
|
106
|
+
}
|
|
107
|
+
function extractAudio(clip, buildDir) {
|
|
108
|
+
const audio = join(buildDir, "audio", `${clip.clip.id}.mp3`);
|
|
109
|
+
mkdirSync(dirname(audio), { recursive: true });
|
|
110
|
+
runFfmpeg(`Extracting the sound of clip ${clip.clip.id}`, buildAudioArgs(clip.file, audio));
|
|
111
|
+
return audio;
|
|
112
|
+
}
|
|
113
|
+
/** The free transcripts of `--placeholder`, written under `placeholder/` for a look. */
|
|
114
|
+
function transcribePlaceholder(film, clips, buildDir) {
|
|
115
|
+
const words = {};
|
|
116
|
+
const speech = {};
|
|
117
|
+
for (const clip of clips) {
|
|
118
|
+
words[clip.clip.id] = clip.clip.role === "talk" ? makePlaceholderTranscript(clip.clip.script, clip.duration) : [];
|
|
119
|
+
writeTranscript(join(buildDir, "placeholder", `${clip.clip.id}.transcript.json`), words[clip.clip.id] ?? []);
|
|
120
|
+
speech[clip.clip.id] = clip.file;
|
|
121
|
+
}
|
|
122
|
+
return { kind: "ready", words, speech };
|
|
123
|
+
}
|
|
124
|
+
/** The paid music step, when the film's music still has to be bought; null when it is cached or free. */
|
|
125
|
+
export function getMusicPaidStep(film, options, seconds) {
|
|
126
|
+
const source = film.music.source;
|
|
127
|
+
if (source.kind === "none" || source.kind === "file")
|
|
128
|
+
return null;
|
|
129
|
+
if (source.kind === "prompt" && existsSync(getPromptMusicPath(film, options)))
|
|
130
|
+
return null;
|
|
131
|
+
return { kind: "music", item: "film", provider: options.providers.music.id, seconds };
|
|
132
|
+
}
|
|
133
|
+
function getPromptMusicPath(film, options) {
|
|
134
|
+
const source = film.music.source;
|
|
135
|
+
const prompt = source.kind === "prompt" ? source.prompt : "";
|
|
136
|
+
const key = footageCacheKey("music", { prompt, provider: options.providers.music.id, model: options.providers.music.model });
|
|
137
|
+
return getFootageCachePath(options.cacheDir, film.id, "music", key);
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Each talking clip's words with times, from the cache or (with `isCommit`) from the speech-to-text provider, the
|
|
141
|
+
* voice cleaned first when the film isolates it. Without `isCommit`, a cache miss returns the dry run: every paid
|
|
142
|
+
* call the film still needs, music included, with its length.
|
|
143
|
+
*/
|
|
144
|
+
export async function transcribeFootage(film, clips, options) {
|
|
145
|
+
const { buildDir, cacheDir, providers, log } = options;
|
|
146
|
+
if (options.isPlaceholder)
|
|
147
|
+
return ok(transcribePlaceholder(film, clips, buildDir));
|
|
148
|
+
const plan = [];
|
|
149
|
+
const steps = [];
|
|
150
|
+
for (const clip of clips) {
|
|
151
|
+
if (clip.clip.role !== "talk")
|
|
152
|
+
continue;
|
|
153
|
+
const clipKey = await getClipKey(clip);
|
|
154
|
+
const isolated = film.isolateVoice ? getFootageCachePath(cacheDir, film.id, "isolated", footageCacheKey("isolated", { clip: clipKey, provider: providers.isolation.id })) : null;
|
|
155
|
+
const transcriptKey = footageCacheKey("transcript", { clip: clipKey, isolated: isolated !== null, provider: providers.stt.id, model: providers.stt.model, language: film.language });
|
|
156
|
+
const transcript = getFootageCachePath(cacheDir, film.id, "transcript", transcriptKey);
|
|
157
|
+
if (isolated !== null && !existsSync(isolated))
|
|
158
|
+
steps.push({ kind: "isolate", item: `clip ${clip.clip.id}`, provider: providers.isolation.id, seconds: clip.duration });
|
|
159
|
+
if (!existsSync(transcript))
|
|
160
|
+
steps.push({ kind: "transcribe", item: `clip ${clip.clip.id}`, provider: providers.stt.id, seconds: clip.duration });
|
|
161
|
+
plan.push({ clip, isolated, transcript });
|
|
162
|
+
}
|
|
163
|
+
if (steps.length > 0 && !options.isCommit) {
|
|
164
|
+
const total = clips.reduce((sum, clip) => sum + clip.duration, 0) + (film.endCard === null ? 0 : END_CARD_SECONDS);
|
|
165
|
+
const music = getMusicPaidStep(film, options, total);
|
|
166
|
+
const all = music === null ? steps : [...steps, music];
|
|
167
|
+
for (const line of describePaidSteps(all))
|
|
168
|
+
log(line);
|
|
169
|
+
log("footage DRY RUN: nothing was sent. Run with --commit to pay, or with --placeholder to rehearse for free.");
|
|
170
|
+
return ok({ kind: "dry-run", steps: all });
|
|
171
|
+
}
|
|
172
|
+
const words = {};
|
|
173
|
+
const speech = {};
|
|
174
|
+
for (const clip of clips) {
|
|
175
|
+
words[clip.clip.id] = [];
|
|
176
|
+
speech[clip.clip.id] = clip.file;
|
|
177
|
+
}
|
|
178
|
+
for (const { clip, isolated, transcript } of plan) {
|
|
179
|
+
const id = clip.clip.id;
|
|
180
|
+
let audio = null;
|
|
181
|
+
const getAudio = () => (audio ??= extractAudio(clip, buildDir));
|
|
182
|
+
let sttInput = null;
|
|
183
|
+
if (isolated !== null) {
|
|
184
|
+
if (!existsSync(isolated)) {
|
|
185
|
+
log(`voice isolation (${providers.isolation.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
|
|
186
|
+
const cleaned = await providers.isolation.isolate({ audio: getAudio() });
|
|
187
|
+
if (!cleaned.ok)
|
|
188
|
+
return err(`clip "${id}": ${cleaned.error}`);
|
|
189
|
+
writeCacheFile(isolated, cleaned.value);
|
|
190
|
+
log(`voice isolation: saved ${isolated} (commit it).`);
|
|
191
|
+
}
|
|
192
|
+
speech[id] = isolated;
|
|
193
|
+
sttInput = isolated;
|
|
194
|
+
}
|
|
195
|
+
const cached = readCachedTranscript(transcript);
|
|
196
|
+
if (!cached.ok)
|
|
197
|
+
return err(cached.error);
|
|
198
|
+
if (cached.words !== null) {
|
|
199
|
+
log(`transcript: clip ${id} from the cache, nothing spent.`);
|
|
200
|
+
words[id] = cached.words;
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
if (sttInput !== null) {
|
|
204
|
+
// The provider sees the clip's id as the file name, as it does for the plain sound.
|
|
205
|
+
sttInput = join(buildDir, "audio", `${id}.isolated.mp3`);
|
|
206
|
+
mkdirSync(dirname(sttInput), { recursive: true });
|
|
207
|
+
copyFileSync(isolated, sttInput);
|
|
208
|
+
}
|
|
209
|
+
log(`speech to text (${providers.stt.id}): clip ${id}, ${clip.duration.toFixed(1)} s.`);
|
|
210
|
+
const result = await providers.stt.transcribe({ audio: sttInput ?? getAudio(), language: film.language });
|
|
211
|
+
if (!result.ok)
|
|
212
|
+
return err(`clip "${id}": ${result.error}`);
|
|
213
|
+
writeTranscript(transcript, result.value.words);
|
|
214
|
+
log(`transcript: clip ${id}, ${result.value.words.length} words, saved ${transcript} (commit it).`);
|
|
215
|
+
words[id] = result.value.words;
|
|
216
|
+
}
|
|
217
|
+
return ok({ kind: "ready", words, speech });
|
|
218
|
+
}
|
|
219
|
+
/** The cut from the transcripts, written to the EDL file (replacing what is there). */
|
|
220
|
+
export function cutFootage(film, clips, words, edlPath) {
|
|
221
|
+
const edl = buildEdl(film.id, clips.map((clip) => ({ id: clip.clip.id, role: clip.clip.role, duration: clip.duration, words: words[clip.clip.id] ?? [] })), film.cut, film.transitions);
|
|
222
|
+
mkdirSync(dirname(edlPath), { recursive: true });
|
|
223
|
+
writeFileSync(edlPath, `${JSON.stringify(edl, null, 2)}\n`);
|
|
224
|
+
return edl;
|
|
225
|
+
}
|
|
226
|
+
/** The EDL file as written by `cutFootage` or edited by hand, checked against the clips. */
|
|
227
|
+
export function readEdlFile(film, clips, edlPath) {
|
|
228
|
+
let json;
|
|
229
|
+
try {
|
|
230
|
+
json = JSON.parse(readFileSync(edlPath, "utf8"));
|
|
231
|
+
}
|
|
232
|
+
catch (error) {
|
|
233
|
+
return err(`${edlPath}: not JSON (${String(error)})`);
|
|
234
|
+
}
|
|
235
|
+
const read = readEdl(json);
|
|
236
|
+
if (!read.ok)
|
|
237
|
+
return err(`${edlPath}: ${read.error}`);
|
|
238
|
+
const problems = findEdlProblems(read.edl, clips.map((clip) => ({ id: clip.clip.id, duration: clip.duration })), film.id);
|
|
239
|
+
return problems.length > 0 ? err(`${edlPath}: ${problems.join("; ")}`) : ok(read.edl);
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* The music file for the cut: a local file as it is, generated music from the cache, or (with `isCommit`) bought
|
|
243
|
+
* from the music provider and cached. Null when the film has none or the rehearsal skips generated music; a paid
|
|
244
|
+
* step when it must be bought and `isCommit` is false.
|
|
245
|
+
*/
|
|
246
|
+
async function getMusic(film, edl, clips, seconds, options) {
|
|
247
|
+
const source = film.music.source;
|
|
248
|
+
if (source.kind === "none")
|
|
249
|
+
return ok(null);
|
|
250
|
+
if (source.kind === "file")
|
|
251
|
+
return existsSync(source.path) ? ok(source.path) : err(`music: no file at ${source.path}`);
|
|
252
|
+
if (options.isPlaceholder)
|
|
253
|
+
return ok(null);
|
|
254
|
+
const { providers, log } = options;
|
|
255
|
+
let path;
|
|
256
|
+
if (source.kind === "prompt") {
|
|
257
|
+
path = getPromptMusicPath(film, options);
|
|
258
|
+
}
|
|
259
|
+
else {
|
|
260
|
+
const clipKeys = {};
|
|
261
|
+
for (const clip of clips)
|
|
262
|
+
clipKeys[clip.clip.id] = await getClipKey(clip);
|
|
263
|
+
const key = footageCacheKey("music", { segments: edl.segments, clips: clipKeys, description: source.description, tags: source.tags, provider: providers.music.id, model: providers.music.model });
|
|
264
|
+
path = getFootageCachePath(options.cacheDir, film.id, "music", key);
|
|
265
|
+
}
|
|
266
|
+
if (existsSync(path)) {
|
|
267
|
+
log(`music: from the cache ${path}, nothing spent.`);
|
|
268
|
+
return ok(path);
|
|
269
|
+
}
|
|
270
|
+
const step = { kind: "music", item: "film", provider: providers.music.id, seconds };
|
|
271
|
+
if (!options.isCommit)
|
|
272
|
+
return ok(step);
|
|
273
|
+
log(`music (${providers.music.id}): ${seconds.toFixed(1)} s.`);
|
|
274
|
+
let bought;
|
|
275
|
+
if (source.kind === "prompt") {
|
|
276
|
+
bought = await providers.music.compose({ prompt: source.prompt, seconds });
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
const preview = join(options.buildDir, "cut-preview.mp4");
|
|
280
|
+
runFfmpeg("The cut preview for Video to Music", buildCutPreviewArgs(edl, Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file])), preview));
|
|
281
|
+
bought = await providers.music.scoreVideo({ video: preview, description: source.description, tags: source.tags });
|
|
282
|
+
}
|
|
283
|
+
if (!bought.ok)
|
|
284
|
+
return err(`music: ${bought.error}`);
|
|
285
|
+
writeCacheFile(path, bought.value);
|
|
286
|
+
log(`music: saved ${path} (commit it).`);
|
|
287
|
+
return ok(path);
|
|
288
|
+
}
|
|
289
|
+
/** Composes the cut and renders it: speech track, ducked music, end card backdrop, assets, hyperframes. */
|
|
290
|
+
export async function renderFootage(film, clips, transcript, edl, options) {
|
|
291
|
+
const { buildDir, log } = options;
|
|
292
|
+
const seconds = getFootageFilmSeconds(film, edl);
|
|
293
|
+
const words = remapWords(edl, transcript.words, film.cut.fillers);
|
|
294
|
+
const music = await getMusic(film, edl, clips, seconds, options);
|
|
295
|
+
if (!music.ok)
|
|
296
|
+
return music;
|
|
297
|
+
if (music.value !== null && typeof music.value !== "string") {
|
|
298
|
+
for (const line of describePaidSteps([music.value]))
|
|
299
|
+
log(line);
|
|
300
|
+
log("footage DRY RUN: the music was not bought. Run with --commit to pay, or with --placeholder to rehearse without it.");
|
|
301
|
+
return ok({ kind: "dry-run", steps: [music.value] });
|
|
302
|
+
}
|
|
303
|
+
const assetsDir = join(buildDir, "assets");
|
|
304
|
+
rmSync(assetsDir, { recursive: true, force: true });
|
|
305
|
+
mkdirSync(assetsDir, { recursive: true });
|
|
306
|
+
const clipFiles = Object.fromEntries(clips.map((clip) => [clip.clip.id, clip.file]));
|
|
307
|
+
runFfmpeg("The speech track", buildSpeechArgs(edl, transcript.speech, join(assetsDir, "speech.mp3")));
|
|
308
|
+
let musicAsset = null;
|
|
309
|
+
if (typeof music.value === "string") {
|
|
310
|
+
const keyframes = duckingKeyframes(words, { volume: film.music.volume, duckedVolume: film.music.duckedVolume });
|
|
311
|
+
runFfmpeg("The music track", buildMusicArgs({ input: music.value, seconds, keyframes, output: join(assetsDir, "music.mp3") }));
|
|
312
|
+
musicAsset = "assets/music.mp3";
|
|
313
|
+
}
|
|
314
|
+
runFfmpeg("The end card's backdrop", buildLastFrameArgs(edl, clipFiles, join(assetsDir, "last.jpg")));
|
|
315
|
+
const fonts = copyFonts(options.brand.fonts, join(assetsDir, "fonts"));
|
|
316
|
+
if (options.brand.logo !== null)
|
|
317
|
+
cpSync(options.brand.logo, join(assetsDir, "logo.svg"));
|
|
318
|
+
const sfx = {};
|
|
319
|
+
mkdirSync(join(assetsDir, "sfx"), { recursive: true });
|
|
320
|
+
for (const [event, source] of Object.entries(options.sfx)) {
|
|
321
|
+
const name = `${event}${extname(source)}`;
|
|
322
|
+
cpSync(source, join(assetsDir, "sfx", name));
|
|
323
|
+
sfx[event] = `assets/sfx/${name}`;
|
|
324
|
+
}
|
|
325
|
+
cpSync(require.resolve("gsap/dist/gsap.min.js"), join(assetsDir, "gsap.min.js"));
|
|
326
|
+
const layout = resolveLayout(film.format, options.layout ?? {});
|
|
327
|
+
const html = composeFootage({
|
|
328
|
+
film,
|
|
329
|
+
edl,
|
|
330
|
+
words,
|
|
331
|
+
frame: layout.frame,
|
|
332
|
+
caption: layout.caption,
|
|
333
|
+
headlineSize: layout.endCard.headlineSize,
|
|
334
|
+
colors: options.brand.colors,
|
|
335
|
+
fonts,
|
|
336
|
+
assets: {
|
|
337
|
+
clips: Object.fromEntries(clips.map((clip) => [clip.clip.id, `clips/${clip.clip.id}.mp4`])),
|
|
338
|
+
speech: "assets/speech.mp3",
|
|
339
|
+
music: musicAsset,
|
|
340
|
+
lastFrame: "assets/last.jpg",
|
|
341
|
+
logo: options.brand.logo === null ? null : "assets/logo.svg",
|
|
342
|
+
sfx,
|
|
343
|
+
},
|
|
344
|
+
brandName: options.brand.name,
|
|
345
|
+
locale: options.brand.locale,
|
|
346
|
+
});
|
|
347
|
+
writeFileSync(join(buildDir, "index.html"), html);
|
|
348
|
+
mkdirSync(dirname(options.output), { recursive: true });
|
|
349
|
+
log(`render: ${edl.segments.length} segments, ${seconds.toFixed(1)} s.`);
|
|
350
|
+
const result = runHyperframes(["render", "--quality", options.quality, "-o", options.output], { cwd: buildDir, stdio: "inherit" });
|
|
351
|
+
if (result.error !== undefined)
|
|
352
|
+
return err(`hyperframes render did not start: ${result.error.message}.`);
|
|
353
|
+
if (result.status !== 0)
|
|
354
|
+
return err(`hyperframes render ended with code ${String(result.status)} (signal ${result.signal ?? "-"}).`);
|
|
355
|
+
return ok({ kind: "rendered", output: options.output, seconds, edlPath: getEdlPath(buildDir, options.isPlaceholder) });
|
|
356
|
+
}
|
|
357
|
+
/**
|
|
358
|
+
* The whole footage film from a `FootageFilm` object, no `marketing.json` needed: prepare, transcribe (or the dry
|
|
359
|
+
* run), cut (reusing an EDL already in the build folder unless `isRecut`), render.
|
|
360
|
+
*/
|
|
361
|
+
export async function renderFootageFilm(film, options) {
|
|
362
|
+
const clips = await prepareFootage(film, options);
|
|
363
|
+
if (!clips.ok)
|
|
364
|
+
return clips;
|
|
365
|
+
const transcript = await transcribeFootage(film, clips.value, options);
|
|
366
|
+
if (!transcript.ok)
|
|
367
|
+
return transcript;
|
|
368
|
+
if (transcript.value.kind === "dry-run")
|
|
369
|
+
return ok(transcript.value);
|
|
370
|
+
const edlPath = getEdlPath(options.buildDir, options.isPlaceholder);
|
|
371
|
+
let edl;
|
|
372
|
+
if (existsSync(edlPath) && options.isRecut !== true) {
|
|
373
|
+
const read = readEdlFile(film, clips.value, edlPath);
|
|
374
|
+
if (!read.ok)
|
|
375
|
+
return read;
|
|
376
|
+
edl = read.value;
|
|
377
|
+
options.log(`cut: ${edlPath} (${edl.segments.length} segments), as it is; softure-marketing footage --step=cut builds it again.`);
|
|
378
|
+
}
|
|
379
|
+
else {
|
|
380
|
+
edl = cutFootage(film, clips.value, transcript.value.words, edlPath);
|
|
381
|
+
options.log(`cut: ${edl.segments.length} segments written to ${edlPath}.`);
|
|
382
|
+
}
|
|
383
|
+
return renderFootage(film, clips.value, transcript.value, edl, options);
|
|
384
|
+
}
|
|
385
|
+
//# sourceMappingURL=pipeline.js.map
|