@morshoto/framekit 0.1.16 → 0.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist-package/adapters/final-cut/typescript/src/connection.js +8 -5
- package/dist-package/adapters/final-cut/typescript/src/fcpxml.js +21 -7
- package/dist-package/adapters/final-cut/typescript/src/timeline-ir-fcpxml.js +110 -28
- package/dist-package/adapters/headless/typescript/src/ffmpeg-renderer.js +120 -12
- package/dist-package/apps/mcp-server/src/headless-projects.js +233 -2
- package/dist-package/apps/mcp-server/src/main.js +6 -3
- package/dist-package/apps/mcp-server/src/server.js +97 -3
- package/dist-package/packages/runtime/src/application/media-analysis-service.js +347 -27
- package/dist-package/packages/runtime/src/application/semantic-media-index-store.js +86 -0
- package/dist-package/packages/runtime/src/index.js +1 -0
- package/dist-package/packages/runtime/src/planning/rough-cut.js +103 -7
- package/dist-package/packages/runtime/src/render/render-contract.js +3 -0
- package/dist-package/packages/runtime/src/runtime.js +5 -2
- package/package.json +1 -1
|
@@ -84,23 +84,23 @@ export class FinalCutConnectionManager {
|
|
|
84
84
|
startAutoConnect() {
|
|
85
85
|
if (this.interval)
|
|
86
86
|
return;
|
|
87
|
-
void this.ensureConnected();
|
|
88
|
-
this.interval = setInterval(() => void this.ensureConnected(), this.pollIntervalMs);
|
|
87
|
+
void this.ensureConnected({ allowEditorLaunch: false });
|
|
88
|
+
this.interval = setInterval(() => void this.ensureConnected({ allowEditorLaunch: false }), this.pollIntervalMs);
|
|
89
89
|
}
|
|
90
90
|
stopAutoConnect() {
|
|
91
91
|
if (this.interval)
|
|
92
92
|
clearInterval(this.interval);
|
|
93
93
|
this.interval = undefined;
|
|
94
94
|
}
|
|
95
|
-
async ensureConnected() {
|
|
95
|
+
async ensureConnected(options = {}) {
|
|
96
96
|
if (this.attempt)
|
|
97
97
|
return this.attempt;
|
|
98
|
-
this.attempt = this.connect().finally(() => {
|
|
98
|
+
this.attempt = this.connect(options.allowEditorLaunch ?? true).finally(() => {
|
|
99
99
|
this.attempt = undefined;
|
|
100
100
|
});
|
|
101
101
|
return this.attempt;
|
|
102
102
|
}
|
|
103
|
-
async connect() {
|
|
103
|
+
async connect(allowEditorLaunch) {
|
|
104
104
|
try {
|
|
105
105
|
this.update({ state: "detecting", lastError: undefined });
|
|
106
106
|
const existing = await this.tryProbe();
|
|
@@ -121,6 +121,9 @@ export class FinalCutConnectionManager {
|
|
|
121
121
|
const editorDetected = await this.detectFinalCut();
|
|
122
122
|
this.update({ editorDetected });
|
|
123
123
|
if (!editorDetected) {
|
|
124
|
+
if (!allowEditorLaunch) {
|
|
125
|
+
return this.fail("FINAL_CUT_AUTO_LAUNCH_SUPPRESSED", "Final Cut Pro is not running; background reconnect will not reopen an explicitly closed editor", "needs-user-action");
|
|
126
|
+
}
|
|
124
127
|
this.update({ state: "launching" });
|
|
125
128
|
await this.launchFinalCut();
|
|
126
129
|
const deadline = Date.now() + Math.min(5_000, this.startupTimeoutMs);
|
|
@@ -837,7 +837,11 @@ export class FcpxmlDocumentAdapter {
|
|
|
837
837
|
const gain = firstChild(node, "adjust-volume");
|
|
838
838
|
const visual = pictureInPictureProperties(node);
|
|
839
839
|
const fades = audioFadeProperties(node);
|
|
840
|
-
const
|
|
840
|
+
const roleAttributeName = kind === "audio" ? "role" : kind === "asset-clip" ? "audioRole" : undefined;
|
|
841
|
+
const roleAttribute = roleAttributeName === undefined ? undefined : attribute(node, roleAttributeName);
|
|
842
|
+
const role = kind === "audio" || roleAttribute === "audio" || roleAttribute === "music"
|
|
843
|
+
? audioRoleFromXml(node, roleAttributeName ?? "role")
|
|
844
|
+
: undefined;
|
|
841
845
|
const sourceStartValue = attribute(node, "start");
|
|
842
846
|
const sourceStartTime = sourceStartValue === undefined ? undefined : parseRational(sourceStartValue);
|
|
843
847
|
const sourceStart = sourceStartTime === undefined ? undefined : rationalSeconds(sourceStartTime);
|
|
@@ -1022,8 +1026,8 @@ function audioFadeProperties(node) {
|
|
|
1022
1026
|
...(fadeOut ? { fadeOut: parseSeconds(attribute(fadeOut, "duration") ?? "0s") } : {}),
|
|
1023
1027
|
};
|
|
1024
1028
|
}
|
|
1025
|
-
function audioRoleFromXml(node) {
|
|
1026
|
-
const role = attribute(node,
|
|
1029
|
+
function audioRoleFromXml(node, attributeName = "role") {
|
|
1030
|
+
const role = attribute(node, attributeName);
|
|
1027
1031
|
if (role === undefined || role === "audio")
|
|
1028
1032
|
return "audio";
|
|
1029
1033
|
if (role === "music")
|
|
@@ -1048,21 +1052,25 @@ function mediaKindCompatible(role, mediaKind) {
|
|
|
1048
1052
|
}
|
|
1049
1053
|
function timelineEntries(spine) {
|
|
1050
1054
|
const entries = [];
|
|
1051
|
-
const visit = (node, parentStart, parentPath) => {
|
|
1055
|
+
const visit = (node, parentStart, parentPath, parentKind) => {
|
|
1052
1056
|
storyEntries(node).forEach(({ kind, node: child }, index) => {
|
|
1053
1057
|
const path = parentPath.length === 0 ? String(index) : `${parentPath}.${index}`;
|
|
1054
|
-
const
|
|
1058
|
+
const localOffset = parseRational(attribute(child, "offset")
|
|
1055
1059
|
?? ((kind === "marker" || kind === "caption") ? attribute(child, "start") : undefined)
|
|
1056
1060
|
?? "0s");
|
|
1061
|
+
const localStart = parentKind === "asset-clip"
|
|
1062
|
+
&& (kind === "title" || kind === "asset-clip" && attribute(child, "audioRole") === "music")
|
|
1063
|
+
? subtractRational(localOffset, parseRational(attribute(node, "start") ?? "0s"))
|
|
1064
|
+
: localOffset;
|
|
1057
1065
|
const durationTime = parseRational(attribute(child, "duration") ?? "0s");
|
|
1058
1066
|
const startTime = addRational(parentStart, localStart);
|
|
1059
1067
|
if (TIMELINE_KINDS.has(kind)) {
|
|
1060
1068
|
entries.push({ kind, node: child, path, startTime, durationTime, parent: node, parentStartTime: parentStart });
|
|
1061
1069
|
}
|
|
1062
|
-
visit(child, startTime, path);
|
|
1070
|
+
visit(child, startTime, path, kind);
|
|
1063
1071
|
});
|
|
1064
1072
|
};
|
|
1065
|
-
visit(spine, { value: "0", timescale: "1" }, "");
|
|
1073
|
+
visit(spine, { value: "0", timescale: "1" }, "", "spine");
|
|
1066
1074
|
return entries;
|
|
1067
1075
|
}
|
|
1068
1076
|
function timelineDuration(entries) {
|
|
@@ -1184,6 +1192,12 @@ function addRational(left, right) {
|
|
|
1184
1192
|
denominator: BigInt(left.timescale) * BigInt(right.timescale),
|
|
1185
1193
|
});
|
|
1186
1194
|
}
|
|
1195
|
+
function subtractRational(left, right) {
|
|
1196
|
+
return rationalTime({
|
|
1197
|
+
numerator: BigInt(left.value) * BigInt(right.timescale) - BigInt(right.value) * BigInt(left.timescale),
|
|
1198
|
+
denominator: BigInt(left.timescale) * BigInt(right.timescale),
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1187
1201
|
function compareRational(left, right) {
|
|
1188
1202
|
const difference = BigInt(left.value) * BigInt(right.timescale)
|
|
1189
1203
|
- BigInt(right.value) * BigInt(left.timescale);
|
|
@@ -3,6 +3,8 @@ import { isAbsolute } from "node:path";
|
|
|
3
3
|
import { pathToFileURL } from "node:url";
|
|
4
4
|
import { timelineIrDigest, validateTimelineIr } from "../../../../packages/runtime/src/index.js";
|
|
5
5
|
export const FRAMEKIT_FCPXML_VERSION = "1.11";
|
|
6
|
+
const BASIC_TITLE_EFFECT_ID = "effect-basic-title";
|
|
7
|
+
const BASIC_TITLE_EFFECT_UID = ".../Titles.localized/Bumper:Opener.localized/Basic Title.localized/Basic Title.moti";
|
|
6
8
|
/**
|
|
7
9
|
* Compiles a complete provider-neutral Timeline IR into a new FCPXML
|
|
8
10
|
* document. The target binding is mandatory because names are not stable
|
|
@@ -41,6 +43,9 @@ export function compileTimelineIrToFcpxml(timeline, options) {
|
|
|
41
43
|
`<fcpxml version="${FRAMEKIT_FCPXML_VERSION}">`,
|
|
42
44
|
" <resources>",
|
|
43
45
|
` <format id="${formatId}" frameDuration="${formatRational(timeline.sequence.frameDuration, "sequence.frameDuration")}" />`,
|
|
46
|
+
...(timeline.sequence.titles?.length
|
|
47
|
+
? [` <effect id="${BASIC_TITLE_EFFECT_ID}" name="Basic Title" uid="${BASIC_TITLE_EFFECT_UID}" />`]
|
|
48
|
+
: []),
|
|
44
49
|
...resources.map((resource) => renderResource(resource, resourceIds.get(resource.id), resourceNames.get(resource.id))),
|
|
45
50
|
" </resources>",
|
|
46
51
|
" <library>",
|
|
@@ -75,7 +80,7 @@ export function compileTimelineIrToFcpxml(timeline, options) {
|
|
|
75
80
|
target: structuredClone(options.target),
|
|
76
81
|
destination,
|
|
77
82
|
resourceIds: Object.fromEntries([...resourceIds.entries()].sort(([left], [right]) => left.localeCompare(right))),
|
|
78
|
-
coverage: materializationCoverage(timeline, destination.mode),
|
|
83
|
+
coverage: materializationCoverage(timeline, destination.mode, elements),
|
|
79
84
|
provenance: {
|
|
80
85
|
source: "framekit-timeline-ir",
|
|
81
86
|
schemaVersion: 1,
|
|
@@ -98,8 +103,9 @@ export function compileTimelineIrToFcpxml(timeline, options) {
|
|
|
98
103
|
}
|
|
99
104
|
function unsupportedFeatures(timeline) {
|
|
100
105
|
const unsupported = new Set();
|
|
101
|
-
if (timeline.sequence.titles?.
|
|
102
|
-
unsupported.add("titles");
|
|
106
|
+
if (timeline.sequence.titles?.some(({ style, position }) => style !== undefined || position !== undefined)) {
|
|
107
|
+
unsupported.add("styled-or-positioned-titles");
|
|
108
|
+
}
|
|
103
109
|
if (timeline.sequence.transitions?.length)
|
|
104
110
|
unsupported.add("transitions");
|
|
105
111
|
for (const occurrence of timeline.sequence.occurrences) {
|
|
@@ -116,7 +122,7 @@ function unsupportedFeatures(timeline) {
|
|
|
116
122
|
}
|
|
117
123
|
return [...unsupported].sort((left, right) => left.localeCompare(right));
|
|
118
124
|
}
|
|
119
|
-
function materializationCoverage(timeline, destinationMode) {
|
|
125
|
+
function materializationCoverage(timeline, destinationMode, elements) {
|
|
120
126
|
const exact = new Set([
|
|
121
127
|
"artifact",
|
|
122
128
|
"project",
|
|
@@ -126,8 +132,7 @@ function materializationCoverage(timeline, destinationMode) {
|
|
|
126
132
|
"primary-storyline-order",
|
|
127
133
|
"provenance",
|
|
128
134
|
]);
|
|
129
|
-
if (
|
|
130
|
-
|| timeline.sequence.storyElements.some(({ attachedTo }) => attachedTo !== undefined))
|
|
135
|
+
if (elements.some(({ parentId }) => parentId !== undefined))
|
|
131
136
|
exact.add("connected-elements");
|
|
132
137
|
if (timeline.sequence.occurrences.some(({ role }) => role !== undefined))
|
|
133
138
|
exact.add("roles");
|
|
@@ -135,6 +140,9 @@ function materializationCoverage(timeline, destinationMode) {
|
|
|
135
140
|
exact.add("gain");
|
|
136
141
|
if (timeline.sequence.occurrences.some(({ enabled }) => enabled !== undefined))
|
|
137
142
|
exact.add("enabled");
|
|
143
|
+
if (timeline.sequence.titles?.length) {
|
|
144
|
+
exact.add("editable-titles");
|
|
145
|
+
}
|
|
138
146
|
if (destinationMode === "versioned")
|
|
139
147
|
exact.add("versioned-destination");
|
|
140
148
|
if (timeline.sequence.markers.length > 0)
|
|
@@ -219,10 +227,13 @@ function renderResource(resource, id, name) {
|
|
|
219
227
|
catch (error) {
|
|
220
228
|
throw new Error(`FCPXML_RESOURCE_SOURCE_UNSUPPORTED: resource ${resource.id} has an invalid file URL: ${error instanceof Error ? error.message : String(error)}`);
|
|
221
229
|
}
|
|
222
|
-
const
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
230
|
+
const videoAttribute = resource.mediaKind === "video" ? ' hasVideo="1"' : ' hasVideo="0"';
|
|
231
|
+
const audioAttribute = resource.mediaKind === "audio"
|
|
232
|
+
? ' hasAudio="1"'
|
|
233
|
+
: resource.metadata?.streams
|
|
234
|
+
? ` hasAudio="${resource.metadata.streams.some((stream) => stream.kind === "audio") ? "1" : "0"}"`
|
|
235
|
+
: "";
|
|
236
|
+
return ` <asset id="${xmlEscape(id)}" name="${xmlEscape(name)}" src="${xmlEscape(src)}"${videoAttribute}${audioAttribute} />`;
|
|
226
237
|
}
|
|
227
238
|
function renderableElements(timeline, resourceIds) {
|
|
228
239
|
const occurrenceIds = new Set(timeline.sequence.occurrences.map(({ id }) => id));
|
|
@@ -244,8 +255,35 @@ function renderableElements(timeline, resourceIds) {
|
|
|
244
255
|
...(element.attachedTo !== undefined ? { parentId: element.attachedTo } : {}),
|
|
245
256
|
});
|
|
246
257
|
}
|
|
258
|
+
for (const title of timeline.sequence.titles ?? []) {
|
|
259
|
+
elements.push(renderableTitle(title, timeline));
|
|
260
|
+
}
|
|
247
261
|
return elements;
|
|
248
262
|
}
|
|
263
|
+
function renderableTitle(title, timeline) {
|
|
264
|
+
const titleEnd = addExactTimes(title.startTime, title.durationTime);
|
|
265
|
+
const parent = timeline.sequence.occurrences.find((occurrence) => {
|
|
266
|
+
if (occurrence.track !== 0 || occurrence.role !== undefined && occurrence.role !== "video")
|
|
267
|
+
return false;
|
|
268
|
+
const occurrenceEnd = addExactTimes(occurrence.startTime, occurrence.durationTime);
|
|
269
|
+
return compareExactTimes(title.startTime, occurrence.startTime) >= 0
|
|
270
|
+
&& compareExactTimes(titleEnd, occurrenceEnd) <= 0;
|
|
271
|
+
});
|
|
272
|
+
if (!parent) {
|
|
273
|
+
throw new Error(`FCPXML_TITLE_ANCHOR_NOT_FOUND: title ${title.id} must fit within one primary video occurrence`);
|
|
274
|
+
}
|
|
275
|
+
return {
|
|
276
|
+
id: title.id,
|
|
277
|
+
kind: "title",
|
|
278
|
+
startTime: title.startTime,
|
|
279
|
+
durationTime: title.durationTime,
|
|
280
|
+
lane: title.lane,
|
|
281
|
+
parentId: parent.id,
|
|
282
|
+
name: title.id,
|
|
283
|
+
resourceId: BASIC_TITLE_EFFECT_ID,
|
|
284
|
+
text: title.text,
|
|
285
|
+
};
|
|
286
|
+
}
|
|
249
287
|
function renderOccurrence(occurrence, timeline, resourceIds, occurrenceIds) {
|
|
250
288
|
if (occurrence.role === "title")
|
|
251
289
|
throw new Error("FCPXML_UNSUPPORTED_OCCURRENCE_ROLE: title");
|
|
@@ -265,13 +303,14 @@ function renderOccurrence(occurrence, timeline, resourceIds, occurrenceIds) {
|
|
|
265
303
|
if (occurrence.attachedTo !== undefined && !occurrenceIds.has(occurrence.attachedTo)) {
|
|
266
304
|
throw new Error(`FCPXML_ATTACHMENT_TARGET_NOT_FOUND: ${occurrence.id} -> ${occurrence.attachedTo}`);
|
|
267
305
|
}
|
|
306
|
+
const parentId = occurrence.attachedTo ?? musicAttachmentTarget(occurrence, timeline);
|
|
268
307
|
return {
|
|
269
308
|
id: occurrence.id,
|
|
270
309
|
kind: "asset-clip",
|
|
271
310
|
startTime: occurrence.startTime,
|
|
272
311
|
durationTime: occurrence.durationTime,
|
|
273
312
|
...(occurrence.track > 0 ? { lane: occurrence.track } : {}),
|
|
274
|
-
...(
|
|
313
|
+
...(parentId !== undefined ? { parentId } : {}),
|
|
275
314
|
name: occurrence.name,
|
|
276
315
|
resourceId: resource,
|
|
277
316
|
...(occurrence.sourceStartTime ? { sourceStartTime: occurrence.sourceStartTime } : {}),
|
|
@@ -280,6 +319,19 @@ function renderOccurrence(occurrence, timeline, resourceIds, occurrenceIds) {
|
|
|
280
319
|
...(occurrence.enabled !== undefined ? { enabled: occurrence.enabled } : {}),
|
|
281
320
|
};
|
|
282
321
|
}
|
|
322
|
+
function musicAttachmentTarget(occurrence, timeline) {
|
|
323
|
+
if (occurrence.role !== "music" || occurrence.track <= 0)
|
|
324
|
+
return undefined;
|
|
325
|
+
const parent = timeline.sequence.occurrences
|
|
326
|
+
.filter((candidate) => candidate.track === 0
|
|
327
|
+
&& candidate.role === "video"
|
|
328
|
+
&& compareExactTimes(candidate.startTime, occurrence.startTime) <= 0)
|
|
329
|
+
.sort((left, right) => compareExactTimes(right.startTime, left.startTime))[0];
|
|
330
|
+
if (!parent) {
|
|
331
|
+
throw new Error(`FCPXML_MUSIC_ANCHOR_NOT_FOUND: ${occurrence.id} must start at or after a primary video occurrence`);
|
|
332
|
+
}
|
|
333
|
+
return parent.id;
|
|
334
|
+
}
|
|
283
335
|
function renderElements(elements) {
|
|
284
336
|
const byParent = new Map();
|
|
285
337
|
const byId = new Map();
|
|
@@ -299,34 +351,52 @@ function renderElements(elements) {
|
|
|
299
351
|
throw new Error(`FCPXML_CYCLIC_ATTACHMENT: ${element.id}`);
|
|
300
352
|
rendered.add(element.id);
|
|
301
353
|
const children = byParent.get(element.id) ?? [];
|
|
354
|
+
const parentElement = element.parentId === undefined ? undefined : byId.get(element.parentId);
|
|
302
355
|
const parentStartTime = element.parentId === undefined
|
|
303
356
|
? { value: "0", timescale: "1" }
|
|
304
|
-
:
|
|
357
|
+
: parentElement?.startTime;
|
|
305
358
|
if (!parentStartTime)
|
|
306
359
|
throw new Error(`FCPXML_ATTACHMENT_TARGET_NOT_FOUND: ${element.id} -> ${element.parentId}`);
|
|
307
360
|
const localStartTime = subtractRational(element.startTime, parentStartTime, `${element.id}.startTime`);
|
|
361
|
+
const offsetTime = parentElement?.kind === "asset-clip"
|
|
362
|
+
&& (element.kind === "title" || element.role === "music")
|
|
363
|
+
&& parentElement.sourceStartTime !== undefined
|
|
364
|
+
? addExactTimes(localStartTime, parentElement.sourceStartTime)
|
|
365
|
+
: localStartTime;
|
|
308
366
|
const attributes = element.kind === "asset-clip"
|
|
309
367
|
? [
|
|
310
368
|
`id="${xmlEscape(element.id)}"`,
|
|
311
369
|
`ref="${xmlEscape(element.resourceId)}"`,
|
|
312
370
|
`name="${xmlEscape(element.name)}"`,
|
|
313
|
-
`offset="${formatRational(
|
|
371
|
+
`offset="${formatRational(offsetTime, `${element.id}.startTime`)}"`,
|
|
314
372
|
...(element.sourceStartTime ? [`start="${formatRational(element.sourceStartTime, `${element.id}.sourceStartTime`)}"`] : []),
|
|
315
373
|
`duration="${formatRational(element.durationTime, `${element.id}.durationTime`)}"`,
|
|
316
374
|
...(element.lane !== undefined ? [`lane="${String(element.lane)}"`] : []),
|
|
317
|
-
...(element.role ? [`
|
|
375
|
+
...(element.role ? [`audioRole="${element.role}"`] : []),
|
|
318
376
|
...(element.enabled !== undefined ? [`enabled="${element.enabled ? "1" : "0"}"`] : []),
|
|
319
377
|
]
|
|
320
|
-
:
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
378
|
+
: element.kind === "title"
|
|
379
|
+
? [
|
|
380
|
+
`id="${xmlEscape(element.id)}"`,
|
|
381
|
+
`ref="${xmlEscape(element.resourceId)}"`,
|
|
382
|
+
`name="${xmlEscape(element.name)}"`,
|
|
383
|
+
`offset="${formatRational(offsetTime, `${element.id}.startTime`)}"`,
|
|
384
|
+
`duration="${formatRational(element.durationTime, `${element.id}.durationTime`)}"`,
|
|
385
|
+
...(element.lane !== undefined ? [`lane="${String(element.lane)}"`] : []),
|
|
386
|
+
]
|
|
387
|
+
: [
|
|
388
|
+
`id="${xmlEscape(element.id)}"`,
|
|
389
|
+
`offset="${formatRational(offsetTime, `${element.id}.startTime`)}"`,
|
|
390
|
+
`duration="${formatRational(element.durationTime, `${element.id}.durationTime`)}"`,
|
|
391
|
+
...(element.lane !== undefined ? [`lane="${String(element.lane)}"`] : []),
|
|
392
|
+
];
|
|
326
393
|
const opening = `<${element.kind} ${attributes.join(" ")}>`;
|
|
327
|
-
const childLines =
|
|
328
|
-
|
|
329
|
-
|
|
394
|
+
const childLines = [
|
|
395
|
+
...(element.gainDb !== undefined
|
|
396
|
+
? [`${indent} <adjust-volume amount="${formatDb(element.gainDb, element.id)}" />`]
|
|
397
|
+
: []),
|
|
398
|
+
...(element.kind === "title" ? [`${indent} <text>${xmlEscape(element.text ?? "")}</text>`] : []),
|
|
399
|
+
];
|
|
330
400
|
const childStart = childLines.length > 0 ? childLines : [];
|
|
331
401
|
if (children.length === 0 && childStart.length === 0) {
|
|
332
402
|
lines.push(`${indent}<${element.kind} ${attributes.join(" ")} />`);
|
|
@@ -346,15 +416,27 @@ function renderElements(elements) {
|
|
|
346
416
|
return lines;
|
|
347
417
|
}
|
|
348
418
|
function compareTimedItems(left, right) {
|
|
349
|
-
const
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
if (difference !== 0n)
|
|
353
|
-
return difference < 0n ? -1 : 1;
|
|
419
|
+
const difference = compareExactTimes(left.startTime, right.startTime);
|
|
420
|
+
if (difference !== 0)
|
|
421
|
+
return difference;
|
|
354
422
|
if ((left.lane ?? 0) !== (right.lane ?? 0))
|
|
355
423
|
return (left.lane ?? 0) - (right.lane ?? 0);
|
|
356
424
|
return left.id.localeCompare(right.id);
|
|
357
425
|
}
|
|
426
|
+
function addExactTimes(left, right) {
|
|
427
|
+
const leftValue = parseExactRational(left);
|
|
428
|
+
const rightValue = parseExactRational(right);
|
|
429
|
+
return {
|
|
430
|
+
value: (leftValue.value * rightValue.timescale + rightValue.value * leftValue.timescale).toString(),
|
|
431
|
+
timescale: (leftValue.timescale * rightValue.timescale).toString(),
|
|
432
|
+
};
|
|
433
|
+
}
|
|
434
|
+
function compareExactTimes(left, right) {
|
|
435
|
+
const leftTime = parseExactRational(left);
|
|
436
|
+
const rightTime = parseExactRational(right);
|
|
437
|
+
const difference = leftTime.value * rightTime.timescale - rightTime.value * leftTime.timescale;
|
|
438
|
+
return difference === 0n ? 0 : difference < 0n ? -1 : 1;
|
|
439
|
+
}
|
|
358
440
|
function formatRational(value, field) {
|
|
359
441
|
const parsed = parseExactRational(value);
|
|
360
442
|
if (parsed.value < 0n)
|
|
@@ -39,6 +39,7 @@ export class FfmpegTimelineRenderer {
|
|
|
39
39
|
"local-media": supportsLocalMedia(timeline) ? "supported" : "unsupported",
|
|
40
40
|
"structural-edits": supportsStructuralEdits(timeline) ? "supported" : "unsupported",
|
|
41
41
|
"audio-gain": supportsAudioGain(timeline) ? "supported" : "unsupported",
|
|
42
|
+
"audio-mixing": supportsAudioMixing(timeline) ? "supported" : "unsupported",
|
|
42
43
|
transform: supportsTransforms(timeline) ? "supported" : "unsupported",
|
|
43
44
|
titles: supportsTitles(timeline) ? "supported" : "unsupported",
|
|
44
45
|
"cross-dissolve": supportsCrossDissolve(timeline) ? "supported" : "unsupported",
|
|
@@ -107,7 +108,12 @@ export class FfmpegTimelineRenderer {
|
|
|
107
108
|
async function prepareRenderModel(plan) {
|
|
108
109
|
const timeline = structuredClone(plan.timeline);
|
|
109
110
|
const sequence = timeline.sequence;
|
|
110
|
-
const occurrences =
|
|
111
|
+
const occurrences = sequence.occurrences
|
|
112
|
+
.filter(isPrimaryVideoOccurrence)
|
|
113
|
+
.sort((left, right) => compareSeconds(left.startTime, right.startTime));
|
|
114
|
+
const audioOccurrences = sequence.occurrences
|
|
115
|
+
.filter(isIndependentAudioOccurrence)
|
|
116
|
+
.sort((left, right) => compareSeconds(left.startTime, right.startTime));
|
|
111
117
|
if (!supportsStructuralEdits(timeline)) {
|
|
112
118
|
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", "markers, captions, story elements, disabled clips, and non-primary tracks are not supported");
|
|
113
119
|
}
|
|
@@ -116,6 +122,9 @@ async function prepareRenderModel(plan) {
|
|
|
116
122
|
}
|
|
117
123
|
if (occurrences.length === 0)
|
|
118
124
|
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", "at least one video occurrence is required");
|
|
125
|
+
if (occurrences.length + audioOccurrences.length !== sequence.occurrences.length) {
|
|
126
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", "occurrence roles and lanes do not form a primary video storyline with connected audio layers");
|
|
127
|
+
}
|
|
119
128
|
const duration = rationalSeconds(sequence.durationTime, "sequence duration");
|
|
120
129
|
const frameRate = 1 / rationalSeconds(sequence.frameDuration, "sequence frame duration");
|
|
121
130
|
if (!Number.isFinite(duration) || duration <= 0 || !Number.isFinite(frameRate) || frameRate <= 0) {
|
|
@@ -130,12 +139,12 @@ async function prepareRenderModel(plan) {
|
|
|
130
139
|
const audioAvailability = new Set();
|
|
131
140
|
let previousEnd = 0;
|
|
132
141
|
for (const occurrence of occurrences) {
|
|
133
|
-
if (occurrence.track !== 0 || occurrence.role === "audio" || occurrence.role === "music" || occurrence.role === "title") {
|
|
134
|
-
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `occurrence role or track is not renderable: ${occurrence.id}`);
|
|
135
|
-
}
|
|
136
142
|
if (occurrence.enabled === false || occurrence.attachedTo) {
|
|
137
143
|
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `occurrence has unsupported state: ${occurrence.id}`);
|
|
138
144
|
}
|
|
145
|
+
if (occurrence.fadeIn !== undefined || occurrence.fadeOut !== undefined) {
|
|
146
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `audio fades are not renderable: ${occurrence.id}`);
|
|
147
|
+
}
|
|
139
148
|
const start = rationalSeconds(occurrence.startTime, `${occurrence.id}.startTime`);
|
|
140
149
|
const clipDuration = rationalSeconds(occurrence.durationTime, `${occurrence.id}.durationTime`);
|
|
141
150
|
if (Math.abs(start - previousEnd) > 1e-7) {
|
|
@@ -145,6 +154,8 @@ async function prepareRenderModel(plan) {
|
|
|
145
154
|
const resource = occurrence.mediaId ? resources.get(occurrence.mediaId) : undefined;
|
|
146
155
|
if (!resource)
|
|
147
156
|
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `occurrence has no registered media: ${occurrence.id}`);
|
|
157
|
+
if (resource.mediaKind !== "video")
|
|
158
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `primary occurrence does not reference video media: ${occurrence.id}`);
|
|
148
159
|
const source = await verifyResource(resource, occurrence, clipDuration);
|
|
149
160
|
const sourcePath = source.path;
|
|
150
161
|
audioAvailability.add(resource.metadata?.streams.some((stream) => stream.kind === "audio") ?? false);
|
|
@@ -154,6 +165,28 @@ async function prepareRenderModel(plan) {
|
|
|
154
165
|
sourceIdentities.push(source);
|
|
155
166
|
}
|
|
156
167
|
}
|
|
168
|
+
for (const occurrence of audioOccurrences) {
|
|
169
|
+
if (occurrence.enabled === false || occurrence.attachedTo) {
|
|
170
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `audio occurrence has unsupported state: ${occurrence.id}`);
|
|
171
|
+
}
|
|
172
|
+
if (occurrence.fadeIn !== undefined || occurrence.fadeOut !== undefined) {
|
|
173
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `audio fades are not renderable: ${occurrence.id}`);
|
|
174
|
+
}
|
|
175
|
+
const start = rationalSeconds(occurrence.startTime, `${occurrence.id}.startTime`);
|
|
176
|
+
const clipDuration = rationalSeconds(occurrence.durationTime, `${occurrence.id}.durationTime`);
|
|
177
|
+
if (clipDuration <= 0 || start + clipDuration > duration + 1e-7) {
|
|
178
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `audio occurrence exceeds the sequence duration: ${occurrence.id}`);
|
|
179
|
+
}
|
|
180
|
+
const resource = occurrence.mediaId ? resources.get(occurrence.mediaId) : undefined;
|
|
181
|
+
if (!resource)
|
|
182
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `audio occurrence has no registered media: ${occurrence.id}`);
|
|
183
|
+
const source = await verifyAudioResource(resource, occurrence, clipDuration);
|
|
184
|
+
if (!sourcePaths.includes(source.path))
|
|
185
|
+
sourcePaths.push(source.path);
|
|
186
|
+
if (!sourceIdentities.some((identity) => identity.path === source.path && identity.digest === source.digest)) {
|
|
187
|
+
sourceIdentities.push(source);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
157
190
|
if (audioAvailability.size > 1) {
|
|
158
191
|
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", "all primary occurrences must agree on audio availability");
|
|
159
192
|
}
|
|
@@ -167,6 +200,7 @@ async function prepareRenderModel(plan) {
|
|
|
167
200
|
return {
|
|
168
201
|
timeline,
|
|
169
202
|
occurrences,
|
|
203
|
+
audioOccurrences,
|
|
170
204
|
resources,
|
|
171
205
|
sourcePaths,
|
|
172
206
|
duration,
|
|
@@ -210,6 +244,38 @@ async function verifyResource(resource, occurrence, duration) {
|
|
|
210
244
|
}
|
|
211
245
|
return { path: sourcePath, digest: actualDigest };
|
|
212
246
|
}
|
|
247
|
+
async function verifyAudioResource(resource, occurrence, duration) {
|
|
248
|
+
if (resource.mediaKind !== "audio" || resource.sourceKind !== "local-file" || !resource.source || !resource.sourceDigest) {
|
|
249
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `resource is not registered local audio: ${resource.id}`);
|
|
250
|
+
}
|
|
251
|
+
const sourcePath = resolve(resource.source);
|
|
252
|
+
let details;
|
|
253
|
+
try {
|
|
254
|
+
details = await lstat(sourcePath);
|
|
255
|
+
}
|
|
256
|
+
catch (error) {
|
|
257
|
+
if (error.code === "ENOENT") {
|
|
258
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_MEDIA_MISSING", `registered source is missing: ${sourcePath}`);
|
|
259
|
+
}
|
|
260
|
+
throw error;
|
|
261
|
+
}
|
|
262
|
+
if (!details.isFile() || details.isSymbolicLink()) {
|
|
263
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_MEDIA_MISSING", `registered source is not a regular file: ${sourcePath}`);
|
|
264
|
+
}
|
|
265
|
+
const actualDigest = await digestFile(sourcePath);
|
|
266
|
+
if (actualDigest !== resource.sourceDigest) {
|
|
267
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_MEDIA_CHANGED", `registered source changed: ${sourcePath}`);
|
|
268
|
+
}
|
|
269
|
+
if (!resource.metadata?.streams.some((stream) => stream.kind === "audio")) {
|
|
270
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `resource is missing audio metadata: ${resource.id}`);
|
|
271
|
+
}
|
|
272
|
+
const sourceStart = occurrence.sourceStartTime ? rationalSeconds(occurrence.sourceStartTime, `${occurrence.id}.sourceStartTime`) : 0;
|
|
273
|
+
const sourceDuration = resource.metadata ? rationalSeconds(resource.metadata.durationTime, `${resource.id}.durationTime`) : 0;
|
|
274
|
+
if (sourceStart < 0 || duration <= 0 || sourceStart + duration > sourceDuration + 1e-7) {
|
|
275
|
+
throw new FfmpegRenderError("HEADLESS_RENDER_UNSUPPORTED", `source audio range exceeds registered media: ${occurrence.id}`);
|
|
276
|
+
}
|
|
277
|
+
return { path: sourcePath, digest: actualDigest };
|
|
278
|
+
}
|
|
213
279
|
async function assertSourcesUnchanged(model) {
|
|
214
280
|
for (const identity of model.sourceIdentities) {
|
|
215
281
|
let details;
|
|
@@ -229,11 +295,18 @@ function buildFfmpegArguments(model, stagingPath, fontFile) {
|
|
|
229
295
|
const inputArguments = [];
|
|
230
296
|
const videoLabels = [];
|
|
231
297
|
const audioLabels = [];
|
|
298
|
+
const musicLabels = [];
|
|
232
299
|
const fps = formatNumber(model.frameRate);
|
|
233
|
-
|
|
300
|
+
const inputIndices = new Map();
|
|
301
|
+
for (const [index, occurrence] of [...model.occurrences, ...model.audioOccurrences].entries()) {
|
|
234
302
|
const resource = model.resources.get(occurrence.mediaId);
|
|
235
303
|
const inputPath = resolve(resource.source);
|
|
236
304
|
inputArguments.push("-i", inputPath);
|
|
305
|
+
inputIndices.set(occurrence.id, index);
|
|
306
|
+
}
|
|
307
|
+
for (const occurrence of model.occurrences) {
|
|
308
|
+
const index = inputIndices.get(occurrence.id);
|
|
309
|
+
const resource = model.resources.get(occurrence.mediaId);
|
|
237
310
|
const sourceStart = occurrence.sourceStartTime ? rationalSeconds(occurrence.sourceStartTime, `${occurrence.id}.sourceStartTime`) : 0;
|
|
238
311
|
const duration = rationalSeconds(occurrence.durationTime, `${occurrence.id}.durationTime`);
|
|
239
312
|
const videoInput = `[${index}:v]`;
|
|
@@ -248,12 +321,27 @@ function buildFfmpegArguments(model, stagingPath, fontFile) {
|
|
|
248
321
|
audioLabels.push(`[${audioLabel}]`);
|
|
249
322
|
}
|
|
250
323
|
}
|
|
324
|
+
for (const occurrence of model.audioOccurrences) {
|
|
325
|
+
const index = inputIndices.get(occurrence.id);
|
|
326
|
+
const sourceStart = occurrence.sourceStartTime ? rationalSeconds(occurrence.sourceStartTime, `${occurrence.id}.sourceStartTime`) : 0;
|
|
327
|
+
const start = rationalSeconds(occurrence.startTime, `${occurrence.id}.startTime`);
|
|
328
|
+
const duration = rationalSeconds(occurrence.durationTime, `${occurrence.id}.durationTime`);
|
|
329
|
+
const gain = occurrence.gainDb ?? 0;
|
|
330
|
+
const delaySamples = Math.round(start * 48_000);
|
|
331
|
+
const audioLabel = `bed-audio${index}`;
|
|
332
|
+
filters.push(`[${index}:a]atrim=start=${formatNumber(sourceStart)}:duration=${formatNumber(duration)},asetpts=PTS-STARTPTS,aresample=48000,aformat=channel_layouts=mono,volume=${formatNumber(Math.pow(10, gain / 20))},adelay=${delaySamples}S:all=1,apad,atrim=duration=${formatNumber(model.duration)},asetpts=PTS-STARTPTS[${audioLabel}]`);
|
|
333
|
+
musicLabels.push(`[${audioLabel}]`);
|
|
334
|
+
}
|
|
251
335
|
const transition = model.timeline.sequence.transitions?.[0];
|
|
252
336
|
const videoBase = addVideoComposition(filters, videoLabels, model, transition);
|
|
253
337
|
const videoOutput = addTitleOverlays(filters, videoBase, model, fontFile);
|
|
254
338
|
let audioOutput;
|
|
255
339
|
if (audioLabels.length > 0)
|
|
256
340
|
audioOutput = addAudioComposition(filters, audioLabels, model, transition);
|
|
341
|
+
if (musicLabels.length > 0) {
|
|
342
|
+
const inputs = [...(audioOutput ? [`[${audioOutput}]`] : []), ...musicLabels];
|
|
343
|
+
audioOutput = addAudioMix(filters, inputs, model.duration);
|
|
344
|
+
}
|
|
257
345
|
const args = ["-hide_banner", "-loglevel", "error", "-nostdin", "-y", ...inputArguments, "-filter_complex", filters.join(";")];
|
|
258
346
|
args.push("-map", `[${videoOutput}]`, "-c:v", "libx264", "-preset", "ultrafast", "-threads", "1", "-pix_fmt", "yuv420p", "-r", fps);
|
|
259
347
|
if (audioOutput) {
|
|
@@ -262,6 +350,13 @@ function buildFfmpegArguments(model, stagingPath, fontFile) {
|
|
|
262
350
|
args.push("-map_metadata", "-1", "-t", formatNumber(model.duration), "-movflags", "+faststart", stagingPath);
|
|
263
351
|
return args;
|
|
264
352
|
}
|
|
353
|
+
function addAudioMix(filters, labels, duration) {
|
|
354
|
+
if (labels.length === 1)
|
|
355
|
+
return labels[0].slice(1, -1);
|
|
356
|
+
const output = "audio-mix";
|
|
357
|
+
filters.push(`${labels.join("")}amix=inputs=${labels.length}:duration=longest:dropout_transition=0:normalize=0,apad,atrim=duration=${formatNumber(duration)},asetpts=PTS-STARTPTS[${output}]`);
|
|
358
|
+
return output;
|
|
359
|
+
}
|
|
265
360
|
function addVideoComposition(filters, labels, model, transition) {
|
|
266
361
|
if (labels.length === 1) {
|
|
267
362
|
const output = "video-base";
|
|
@@ -345,16 +440,28 @@ function supportsLocalMedia(timeline) {
|
|
|
345
440
|
});
|
|
346
441
|
}
|
|
347
442
|
function supportsStructuralEdits(timeline) {
|
|
443
|
+
const primaryOccurrences = timeline.sequence.occurrences.filter(isPrimaryVideoOccurrence);
|
|
348
444
|
return timeline.sequence.storyElements.length === 0
|
|
349
445
|
&& timeline.sequence.markers.length === 0
|
|
350
446
|
&& timeline.sequence.captions.length === 0
|
|
351
|
-
&&
|
|
352
|
-
&& timeline.sequence.occurrences.
|
|
353
|
-
|
|
354
|
-
&& occurrence.role !== "music"
|
|
355
|
-
&& occurrence.role !== "title"
|
|
447
|
+
&& primaryOccurrences.length > 0
|
|
448
|
+
&& primaryOccurrences.length + timeline.sequence.occurrences.filter(isIndependentAudioOccurrence).length === timeline.sequence.occurrences.length
|
|
449
|
+
&& timeline.sequence.occurrences.every((occurrence) => (isPrimaryVideoOccurrence(occurrence) || isIndependentAudioOccurrence(occurrence))
|
|
356
450
|
&& occurrence.enabled !== false
|
|
357
|
-
&& !occurrence.attachedTo
|
|
451
|
+
&& !occurrence.attachedTo
|
|
452
|
+
&& occurrence.fadeIn === undefined
|
|
453
|
+
&& occurrence.fadeOut === undefined);
|
|
454
|
+
}
|
|
455
|
+
function supportsAudioMixing(timeline) {
|
|
456
|
+
return timeline.sequence.occurrences
|
|
457
|
+
.filter((occurrence) => occurrence.role === "audio" || occurrence.role === "music")
|
|
458
|
+
.every(isIndependentAudioOccurrence);
|
|
459
|
+
}
|
|
460
|
+
function isPrimaryVideoOccurrence(occurrence) {
|
|
461
|
+
return occurrence.track === 0 && (occurrence.role === undefined || occurrence.role === "video");
|
|
462
|
+
}
|
|
463
|
+
function isIndependentAudioOccurrence(occurrence) {
|
|
464
|
+
return occurrence.track > 0 && (occurrence.role === "audio" || occurrence.role === "music");
|
|
358
465
|
}
|
|
359
466
|
function supportsAudioGain(timeline) {
|
|
360
467
|
return timeline.sequence.occurrences.every((occurrence) => occurrence.gainDb === undefined || Number.isFinite(occurrence.gainDb));
|
|
@@ -373,9 +480,10 @@ function supportsTitles(timeline) {
|
|
|
373
480
|
}
|
|
374
481
|
function supportsCrossDissolve(timeline) {
|
|
375
482
|
const transitions = timeline.sequence.transitions ?? [];
|
|
483
|
+
const primaryCount = timeline.sequence.occurrences.filter(isPrimaryVideoOccurrence).length;
|
|
376
484
|
return transitions.length <= 1
|
|
377
485
|
&& transitions.every((transition) => transition.kind === "cross-dissolve")
|
|
378
|
-
&& (transitions.length === 0 ||
|
|
486
|
+
&& (transitions.length === 0 || primaryCount === 2);
|
|
379
487
|
}
|
|
380
488
|
async function assertOutputPath(outputPath, overwrite, sourcePaths) {
|
|
381
489
|
if (sourcePaths.includes(outputPath)) {
|