frameyard 0.7.0 → 0.9.0

package/dist/renderer/timeline.js

@@ -8,9 +8,13 @@ import { validateHtmlComposition } from "../html/validate.js";
 import { startPreviewServer } from "../server/server.js";
 import { inspectTimelineAudio } from "./audio.js";
 import { openCompositionSession } from "./browser.js";
-import { assertContainerCodec, assertFfmpegCapabilities, compositeTimelineFrame, compositeTimelineSegment, concatTimelineSegments, CONTAINER_EXTENSION, defaultCodecForContainer, ENCODER_SESSION_LIMIT, encodePngSequenceToFfv1, mixAudioTimeline, muxVideoWithAudio, resolveVideoEncoder, } from "./ffmpeg.js";
+import { assertContainerCodec, assertFfmpegCapabilities, compositeTimelineFrame, compositeTimelineSegment, concatTimelineSegments, CONTAINER_EXTENSION, defaultCodecForContainer, ENCODER_SESSION_LIMIT, encodePngSequenceToFfv1, mixAudioTimeline, muxVideoWithAudio, resolveVideoEncoder, sceneLayerToCompositeLayer, } from "./ffmpeg.js";
 import { inspectTranscriptClips } from "./transcripts.js";
+import { evaluatePresentation } from "../timeline/keyframes.js";
+import { transitionProgress } from "../timeline/transitions.js";
+import { KEYFRAME_CHANNELS } from "../timeline/schema.js";
 import { planTimelineRender } from "../timeline/plan.js";
+import { projectScene, transitionSpans } from "../timeline/scene.js";
 import { resolveProjectTimeline, } from "../timeline/resolve.js";
 export const MAX_TIMELINE_WORKERS = 16;
 function validateWorkers(value) {
@@ -136,19 +140,6 @@ function compositionDiagnostic(diagnostic, timelineId, clip, compositionId) {
 function clampVolume(value) {
     return Math.min(1, Math.max(0, value));
 }
-function visualSourceSize(clip, timeline) {
-    const source = clip.source;
-    if (source.kind === "composition") {
-        return { width: source.metadata.width, height: source.metadata.height };
-    }
-    if (source.kind === "video" || source.kind === "image") {
-        if (source.width === undefined || source.height === undefined) {
-            throw new FrameyardError("TIMELINE_MEDIA_INVALID", `Media ${source.src} is missing width/height metadata for clip "${clip.id}".`);
-        }
-        return { width: source.width, height: source.height };
-    }
-    return { width: timeline.format.width, height: timeline.format.height };
-}
 function mapCompositionAudio(clip, audio, fps, window) {
     if (window.muted) {
         return null;
@@ -178,11 +169,13 @@ function mapCompositionAudio(clip, audio, fps, window) {
         trimStart: audio.trimStart + (effectiveStart - audio.startFrame) / fps,
         trimDuration: duration,
         volume: clampVolume(audio.volume * window.gain),
+        ...(window.envelopes.length > 0 ? { envelopes: window.envelopes } : {}),
     };
 }
 function collectTimelineAudio(timeline, window, clips, transcripts) {
     const fps = timeline.format.fps;
     const trackById = new Map(timeline.tracks.map((track) => [track.id, track]));
+    const spansByClip = transitionSpans(timeline);
     for (const clip of timeline.clips) {
         const source = clip.source;
         const mappedStart = clip.start + window.frameOffset;
@@ -194,6 +187,13 @@ function collectTimelineAudio(timeline, window, clips, transcripts) {
         }
         const clipGain = window.gain * (clip.audio?.gain ?? 1);
         const clipMuted = window.muted || (clip.audio?.muted ?? false);
+        const ownEnvelopes = (spansByClip.get(clip.id) ?? []).map((span) => ({
+            startFrame: span.start + window.frameOffset,
+            endFrame: span.end + window.frameOffset,
+            ...(span.easing !== undefined ? { easing: span.easing } : {}),
+            role: span.role,
+        }));
+        const envelopes = ownEnvelopes.length > 0 ? [...window.envelopes, ...ownEnvelopes] : window.envelopes;
         if (source.kind === "composition") {
             for (const transcript of source.transcriptClips) {
                 transcripts.set(`${transcript.src}\u0000${transcript.words}`, transcript);
@@ -202,13 +202,13 @@ function collectTimelineAudio(timeline, window, clips, transcripts) {
                 continue;
             }
             for (const audio of source.audioClips) {
-                const mapped = mapCompositionAudio(clip, audio, fps, { ...window, gain: clipGain });
+                const mapped = mapCompositionAudio(clip, audio, fps, { ...window, gain: clipGain, envelopes });
                 if (mapped) {
                     clips.push(mapped);
                 }
             }
             for (const audio of source.embeddedAudioClips) {
-                const mapped = mapCompositionAudio(clip, audio, fps, { ...window, gain: clipGain });
+                const mapped = mapCompositionAudio(clip, audio, fps, { ...window, gain: clipGain, envelopes });
                 if (mapped) {
                     clips.push({ ...mapped, allowSilent: true });
                 }
@@ -222,6 +222,7 @@ function collectTimelineAudio(timeline, window, clips, transcripts) {
                 endFrame,
                 gain: clipGain,
                 muted: clipMuted,
+                envelopes,
             }, clips, transcripts);
             continue;
         }
@@ -233,6 +234,7 @@ function collectTimelineAudio(timeline, window, clips, transcripts) {
                     trimStart: clip.sourceIn / fps + (startFrame - mappedStart) / fps,
                     trimDuration: (endFrame - startFrame) / fps,
                     volume: clampVolume(clipGain),
+                    ...(envelopes.length > 0 ? { envelopes } : {}),
                 });
             }
             continue;
@@ -248,6 +250,7 @@ function collectTimelineAudio(timeline, window, clips, transcripts) {
                 trimStart: clip.sourceIn / fps + (startFrame - mappedStart) / fps,
                 trimDuration: (endFrame - startFrame) / fps,
                 volume: clampVolume(clipGain * (track.gain ?? 1)),
+                ...(envelopes.length > 0 ? { envelopes } : {}),
             });
         }
     }
@@ -255,7 +258,7 @@ function collectTimelineAudio(timeline, window, clips, transcripts) {
 function buildAudioClips(timeline) {
     const clips = [];
     const transcripts = new Map();
-    collectTimelineAudio(timeline, { frameOffset: 0, startFrame: 0, endFrame: timeline.durationInFrames, gain: 1, muted: false }, clips, transcripts);
+    collectTimelineAudio(timeline, { frameOffset: 0, startFrame: 0, endFrame: timeline.durationInFrames, gain: 1, muted: false, envelopes: [] }, clips, transcripts);
     return { clips, transcripts: [...transcripts.values()] };
 }
 /** Resolved timeline audio for preview surfaces. */
@@ -269,6 +272,58 @@ export function resolveTimelineAudio(timeline) {
         transcripts,
     };
 }
+async function captureStaticCompositionLayer(options) {
+    const { project, timeline, clip, source, serverUrl, scratchRoot, infoCache, staticCache } = options;
+    const propsKey = JSON.stringify(clip.props ?? null);
+    const cacheKey = `${source.entry}\u0000${propsKey}`;
+    const cached = staticCache.get(cacheKey);
+    if (cached) {
+        return cached;
+    }
+    const pending = (async () => {
+        let validation = infoCache.get(cacheKey);
+        if (!validation) {
+            validation = await validateHtmlComposition({
+                projectRoot: project.root,
+                entry: source.entry,
+                ...(clip.props !== undefined ? { props: clip.props } : {}),
+                ...(project.format ? { format: project.format } : {}),
+            });
+            infoCache.set(cacheKey, validation);
+        }
+        if (!validation.info) {
+            throw new FrameyardError("TIMELINE_CLIP_INVALID", `Composition "${source.id}" could not be loaded for clip "${clip.id}".`, { diagnostics: validation.diagnostics });
+        }
+        const session = await openCompositionSession({
+            serverUrl,
+            entry: source.entry,
+            props: clip.props,
+            info: validation.info,
+            staticDiagnostics: validation.diagnostics,
+            transparentBackground: true,
+        });
+        try {
+            throwIfAborted(options.signal);
+            await session.seek(0);
+            const png = await session.capture();
+            const filePath = path.join(scratchRoot, `static-${randomUUID()}.png`);
+            await writeFile(filePath, png);
+            const diagnostics = (await session.readDiagnostics()).map((diagnostic) => compositionDiagnostic(diagnostic, timeline.id, clip, source.id));
+            return { path: filePath, diagnostics };
+        }
+        finally {
+            await session.close();
+        }
+    })();
+    staticCache.set(cacheKey, pending);
+    try {
+        return await pending;
+    }
+    catch (error) {
+        staticCache.delete(cacheKey);
+        throw error;
+    }
+}
 async function captureCompositionLayer(options) {
     const { project, timeline, clip, source, segment, layerIndex, serverUrl, segmentDir, infoCache } = options;
     const propsKey = JSON.stringify(clip.props ?? null);
@@ -279,6 +334,7 @@ async function captureCompositionLayer(options) {
             projectRoot: project.root,
             entry: source.entry,
             ...(clip.props !== undefined ? { props: clip.props } : {}),
+            ...(project.format ? { format: project.format } : {}),
         });
         infoCache.set(cacheKey, validation);
     }
@@ -341,6 +397,7 @@ async function renderNestedTimelineRange(options) {
             serverUrl: options.serverUrl,
             scratchRoot: options.scratchRoot,
             infoCache: options.infoCache,
+            staticCache: options.staticCache,
             codec: "ffv1",
             background: options.timeline.background ?? "transparent",
             prefix,
@@ -377,26 +434,23 @@ async function processTimelineSegment(options) {
         throwIfAborted(options.signal);
         const clip = layer.clip;
         const source = clip.source;
-        const transform = clip.transform ?? {
-            fit: "contain",
-            x: 0,
-            y: 0,
-            scale: 1,
-            rotation: 0,
-            opacity: 1,
-        };
-        const base = {
-            fit: transform.fit,
-            x: transform.x,
-            y: transform.y,
-            ...(transform.width !== undefined && transform.height !== undefined
-                ? { width: transform.width, height: transform.height }
-                : {}),
-            scale: transform.scale,
-            rotation: transform.rotation,
-            opacity: transform.opacity,
-        };
         if (source.kind === "composition") {
+            if (layer.capture === "still") {
+                const captured = await captureStaticCompositionLayer({
+                    project,
+                    timeline,
+                    clip,
+                    source,
+                    serverUrl,
+                    scratchRoot,
+                    infoCache,
+                    staticCache: options.staticCache,
+                    ...(options.signal ? { signal: options.signal } : {}),
+                });
+                diagnostics.push(...captured.diagnostics);
+                layers.push(sceneLayerToCompositeLayer(layer, { path: captured.path, input: "image" }));
+                continue;
+            }
             const captured = await captureCompositionLayer({
                 project,
                 timeline,
@@ -410,25 +464,15 @@ async function processTimelineSegment(options) {
                 ...(options.signal ? { signal: options.signal } : {}),
             });
             diagnostics.push(...captured.diagnostics);
-            layers.push({
-                kind: "composition",
-                path: captured.path,
-                sourceWidth: source.metadata.width,
-                sourceHeight: source.metadata.height,
-                ...base,
-            });
+            layers.push(sceneLayerToCompositeLayer(layer, { path: captured.path, input: "sequence" }));
             continue;
         }
         if (source.kind === "video" || source.kind === "image") {
-            const size = visualSourceSize(clip, timeline);
-            layers.push({
-                kind: source.kind,
+            layers.push(sceneLayerToCompositeLayer(layer, {
                 path: source.absolutePath,
-                sourceWidth: size.width,
-                sourceHeight: size.height,
-                ...base,
+                input: source.kind,
                 ...(source.kind === "video" ? { sourceStartFrame: layer.sourceStart } : {}),
-            });
+            }));
             continue;
         }
         if (source.kind === "timeline") {
@@ -440,17 +484,12 @@ async function processTimelineSegment(options) {
                 serverUrl,
                 scratchRoot,
                 infoCache,
+                staticCache: options.staticCache,
                 depth: depth + 1,
                 ...(options.signal ? { signal: options.signal } : {}),
             });
             diagnostics.push(...nested.diagnostics);
-            layers.push({
-                kind: "timeline",
-                path: nested.path,
-                sourceWidth: source.timeline.format.width,
-                sourceHeight: source.timeline.format.height,
-                ...base,
-            });
+            layers.push(sceneLayerToCompositeLayer(layer, { path: nested.path, input: "sequence" }));
             continue;
         }
         throw new FrameyardError("TIMELINE_INVALID", `Unsupported visual source for clip "${clip.id}".`);
@@ -466,6 +505,7 @@ async function processTimelineSegment(options) {
         background,
         layers,
         codec,
+        frameOffset: segment.start,
         ...(options.container !== undefined ? { container: options.container } : {}),
         ...(options.encoder !== undefined ? { encoder: options.encoder } : {}),
         ...(options.signal ? { signal: options.signal } : {}),
@@ -473,6 +513,51 @@ async function processTimelineSegment(options) {
     await rm(segmentDir, { recursive: true, force: true });
     return { path: segmentPath, diagnostics };
 }
+function checkKeyframeEvaluation(timeline, segments) {
+    for (const segment of segments) {
+        for (const layer of segment.layers) {
+            const keyframes = layer.keyframes;
+            if (!keyframes) {
+                continue;
+            }
+            for (let frame = segment.start; frame < segment.end; frame += 1) {
+                const values = evaluatePresentation(layer.transform, keyframes, frame - layer.clip.start);
+                for (const channel of KEYFRAME_CHANNELS) {
+                    if (!Number.isFinite(values[channel])) {
+                        throw new FrameyardError("TIMELINE_CLIP_INVALID", `Clip "${layer.clip.id}" keyframes.${channel} produced a non-finite value at timeline frame ${frame}.`, {
+                            details: {
+                                timeline: timeline.id,
+                                clip: layer.clip.id,
+                                track: layer.clip.track,
+                                channel,
+                                frame: frame - layer.clip.start,
+                                timelineFrame: frame,
+                            },
+                        });
+                    }
+                }
+            }
+        }
+    }
+}
+function checkTransitionEvaluation(timeline) {
+    for (const transition of timeline.transitions) {
+        for (let frame = transition.start; frame < transition.end; frame += 1) {
+            const progress = transitionProgress(transition, frame);
+            if (!Number.isFinite(progress)) {
+                throw new FrameyardError("TIMELINE_TRANSITION_INVALID", `Transition "${transition.from}" → "${transition.to}" produced a non-finite progress at timeline frame ${frame}.`, {
+                    details: {
+                        timeline: timeline.id,
+                        track: transition.track,
+                        from: transition.from,
+                        to: transition.to,
+                        timelineFrame: frame,
+                    },
+                });
+            }
+        }
+    }
+}
 async function checkResolvedTimelineRange(options) {
     if (options.context.depth > 32) {
         throw new FrameyardError("TIMELINE_INVALID", "Nested timeline depth exceeds 32.");
@@ -480,6 +565,8 @@ async function checkResolvedTimelineRange(options) {
     const { timeline, startFrame, endFrame, context } = options;
     const plan = planTimelineRender(timeline, { startFrame, endFrame });
     const diagnostics = [];
+    checkKeyframeEvaluation(timeline, plan.segments);
+    checkTransitionEvaluation(timeline);
     const targets = new Map();
     for (const segment of plan.segments) {
         for (const layer of segment.layers) {
@@ -493,6 +580,7 @@ async function checkResolvedTimelineRange(options) {
                     projectRoot: context.project.root,
                     entry: source.entry,
                     ...(layer.clip.props !== undefined ? { props: layer.clip.props } : {}),
+                    ...(context.project.format ? { format: context.project.format } : {}),
                 });
                 if (!validation.info) {
                     throw new FrameyardError("TIMELINE_CLIP_INVALID", `Composition "${source.id}" could not be loaded for clip "${layer.clip.id}".`, { diagnostics: validation.diagnostics });
@@ -512,6 +600,9 @@ async function checkResolvedTimelineRange(options) {
                 staticDiagnostics: target.validation.diagnostics,
             });
             sessions.set(key, session);
+            if (target.source.metadata.static === true) {
+                await session.seek(0);
+            }
         }
         for (const segment of plan.segments) {
             for (let frame = segment.start; frame < segment.end; frame += 1) {
@@ -521,6 +612,9 @@ async function checkResolvedTimelineRange(options) {
                         continue;
                     }
                     const source = layer.clip.source;
+                    if (source.metadata.static === true) {
+                        continue;
+                    }
                     const key = `${source.entry}\u0000${JSON.stringify(layer.clip.props ?? null)}`;
                     const session = sessions.get(key);
                     if (!session) {
@@ -579,7 +673,7 @@ export async function checkTimeline(options) {
     const startedAt = performance.now();
     const projectTimeline = options.project.timelines.get(options.timelineId);
     if (!projectTimeline) {
-        throw new FrameyardError("TIMELINE_NOT_FOUND", `Timeline "${options.timelineId}" is not listed in video.config.js.`);
+        throw new FrameyardError("TIMELINE_NOT_FOUND", `Timeline "${options.timelineId}" is not listed in fy.config.js.`);
     }
     const resolved = await resolveProjectTimeline({
         project: options.project,
@@ -655,6 +749,7 @@ export async function checkTimeline(options) {
         framesChecked: timeline.durationInFrames,
         tracks: timeline.tracks.length,
         clips: timeline.clips.length,
+        transitions: timeline.transitions.length,
         audio: { clips: audioInspection.clips.length, details: audioInspection.clips },
         transcripts: {
             clips: transcripts.clips.length,
@@ -669,40 +764,18 @@ async function renderResolvedTimelineFrame(options) {
         throw new FrameyardError("TIMELINE_INVALID", "Nested timeline depth exceeds 32.");
     }
     const { timeline, frame, context } = options;
-    const plan = planTimelineRender(timeline, { startFrame: frame, endFrame: frame + 1 });
-    const segment = plan.segments[0];
-    if (!segment) {
-        throw new FrameyardError("FRAME_OUT_OF_RANGE", `Frame ${frame} does not belong to a timeline segment.`);
-    }
+    const scene = projectScene(timeline, frame, frame + 1);
     const diagnostics = [];
     const layers = [];
-    for (const [layerIndex, layer] of segment.layers.entries()) {
+    for (const [layerIndex, layer] of scene.layers.entries()) {
         const clip = layer.clip;
         const source = clip.source;
-        const transform = clip.transform ?? {
-            fit: "contain",
-            x: 0,
-            y: 0,
-            scale: 1,
-            rotation: 0,
-            opacity: 1,
-        };
-        const base = {
-            fit: transform.fit,
-            x: transform.x,
-            y: transform.y,
-            ...(transform.width !== undefined && transform.height !== undefined
-                ? { width: transform.width, height: transform.height }
-                : {}),
-            scale: transform.scale,
-            rotation: transform.rotation,
-            opacity: transform.opacity,
-        };
         if (source.kind === "composition") {
             const validation = await validateHtmlComposition({
                 projectRoot: context.project.root,
                 entry: source.entry,
                 ...(clip.props !== undefined ? { props: clip.props } : {}),
+                ...(context.project.format ? { format: context.project.format } : {}),
             });
             if (!validation.info) {
                 throw new FrameyardError("TIMELINE_CLIP_INVALID", `Composition "${source.id}" could not be loaded for clip "${clip.id}".`, { diagnostics: validation.diagnostics });
@@ -727,25 +800,15 @@ async function renderResolvedTimelineFrame(options) {
             finally {
                 await session.close();
             }
-            layers.push({
-                kind: "image",
-                path: framePath,
-                sourceWidth: source.metadata.width,
-                sourceHeight: source.metadata.height,
-                ...base,
-            });
+            layers.push(sceneLayerToCompositeLayer(layer, { path: framePath, input: "image" }));
             continue;
         }
         if (source.kind === "video" || source.kind === "image") {
-            const size = visualSourceSize(clip, timeline);
-            layers.push({
-                kind: source.kind,
+            layers.push(sceneLayerToCompositeLayer(layer, {
                 path: source.absolutePath,
-                sourceWidth: size.width,
-                sourceHeight: size.height,
-                ...base,
+                input: source.kind,
                 ...(source.kind === "video" ? { sourceStartFrame: layer.sourceStart } : {}),
-            });
+            }));
             continue;
         }
         if (source.kind === "timeline") {
@@ -758,13 +821,7 @@ async function renderResolvedTimelineFrame(options) {
                 context: { ...context, depth: context.depth + 1 },
             });
             diagnostics.push(...nestedDiagnostics);
-            layers.push({
-                kind: "image",
-                path: nestedPath,
-                sourceWidth: source.timeline.format.width,
-                sourceHeight: source.timeline.format.height,
-                ...base,
-            });
+            layers.push(sceneLayerToCompositeLayer(layer, { path: nestedPath, input: "image" }));
             continue;
         }
         throw new FrameyardError("TIMELINE_INVALID", `Unsupported visual source for clip "${clip.id}".`);
@@ -776,6 +833,7 @@ async function renderResolvedTimelineFrame(options) {
         fps: timeline.format.fps,
         background: options.background,
         layers,
+        frameOffset: frame,
         ...(context.signal ? { signal: context.signal } : {}),
     });
     return diagnostics;
@@ -808,7 +866,7 @@ export async function captureTimelineStills(options) {
     const startedAt = performance.now();
     const projectTimeline = options.project.timelines.get(options.timelineId);
     if (!projectTimeline) {
-        throw new FrameyardError("TIMELINE_NOT_FOUND", `Timeline "${options.timelineId}" is not listed in video.config.js.`);
+        throw new FrameyardError("TIMELINE_NOT_FOUND", `Timeline "${options.timelineId}" is not listed in fy.config.js.`);
     }
     const resolved = await resolveProjectTimeline({
         project: options.project,
@@ -919,7 +977,7 @@ export async function renderTimeline(options) {
     const workers = clampTimelineWorkers(requestedWorkers, encoder);
     const projectTimeline = options.project.timelines.get(options.timelineId);
     if (!projectTimeline) {
-        throw new FrameyardError("TIMELINE_NOT_FOUND", `Timeline "${options.timelineId}" is not listed in video.config.js.`);
+        throw new FrameyardError("TIMELINE_NOT_FOUND", `Timeline "${options.timelineId}" is not listed in fy.config.js.`);
     }
     const resolved = await resolveProjectTimeline({
         project: options.project,
@@ -940,6 +998,7 @@ export async function renderTimeline(options) {
     const scratchRoot = await mkdtemp(path.join(scratchBase, "frameyard-timeline-"));
     const server = await startPreviewServer({ projectRoot: options.project.root, port: 0 });
     const infoCache = new Map();
+    const staticCache = new Map();
     const diagnostics = [...resolved.diagnostics];
     if (workers < requestedWorkers) {
         diagnostics.push({
@@ -999,6 +1058,7 @@ export async function renderTimeline(options) {
                     serverUrl: server.url,
                     scratchRoot,
                     infoCache,
+                    staticCache,
                     codec,
                     container,
                     encoder,
@@ -1040,6 +1100,7 @@ export async function renderTimeline(options) {
                     trimStart: asset.clip.trimStart,
                     ...(asset.clip.trimDuration !== undefined ? { trimDuration: asset.clip.trimDuration } : {}),
                     volume: asset.clip.volume,
+                    ...(asset.clip.envelopes !== undefined ? { volumeEnvelopes: asset.clip.envelopes } : {}),
                 })),
                 fps: timeline.format.fps,
                 durationSeconds: total / timeline.format.fps,
@@ -1073,6 +1134,7 @@ export async function renderTimeline(options) {
             framesRendered: total,
             segments: plan.segments.length,
             workers,
+            transitions: timeline.transitions.length,
             codec,
             container,
             encoder,