@nodetool-ai/video-nodes 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -0
- package/dist/.nodetool-build-stamp +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/nodes/ffmpeg-helpers.d.ts +77 -0
- package/dist/nodes/ffmpeg-helpers.d.ts.map +1 -0
- package/dist/nodes/ffmpeg-helpers.js +220 -0
- package/dist/nodes/ffmpeg-helpers.js.map +1 -0
- package/dist/nodes/lib-video-download.d.ts +29 -0
- package/dist/nodes/lib-video-download.d.ts.map +1 -0
- package/dist/nodes/lib-video-download.js +353 -0
- package/dist/nodes/lib-video-download.js.map +1 -0
- package/dist/nodes/model3d/analysis.d.ts +16 -0
- package/dist/nodes/model3d/analysis.d.ts.map +1 -0
- package/dist/nodes/model3d/analysis.js +38 -0
- package/dist/nodes/model3d/analysis.js.map +1 -0
- package/dist/nodes/model3d/base.d.ts +28 -0
- package/dist/nodes/model3d/base.d.ts.map +1 -0
- package/dist/nodes/model3d/base.js +20 -0
- package/dist/nodes/model3d/base.js.map +1 -0
- package/dist/nodes/model3d/boolean-ops.d.ts +2 -0
- package/dist/nodes/model3d/boolean-ops.d.ts.map +1 -0
- package/dist/nodes/model3d/boolean-ops.js +172 -0
- package/dist/nodes/model3d/boolean-ops.js.map +1 -0
- package/dist/nodes/model3d/cleanup.d.ts +56 -0
- package/dist/nodes/model3d/cleanup.d.ts.map +1 -0
- package/dist/nodes/model3d/cleanup.js +144 -0
- package/dist/nodes/model3d/cleanup.js.map +1 -0
- package/dist/nodes/model3d/convert.d.ts +27 -0
- package/dist/nodes/model3d/convert.d.ts.map +1 -0
- package/dist/nodes/model3d/convert.js +66 -0
- package/dist/nodes/model3d/convert.js.map +1 -0
- package/dist/nodes/model3d/defaults.d.ts +34 -0
- package/dist/nodes/model3d/defaults.d.ts.map +1 -0
- package/dist/nodes/model3d/defaults.js +28 -0
- package/dist/nodes/model3d/defaults.js.map +1 -0
- package/dist/nodes/model3d/document-ops.d.ts +4 -0
- package/dist/nodes/model3d/document-ops.d.ts.map +1 -0
- package/dist/nodes/model3d/document-ops.js +40 -0
- package/dist/nodes/model3d/document-ops.js.map +1 -0
- package/dist/nodes/model3d/generation.d.ts +56 -0
- package/dist/nodes/model3d/generation.d.ts.map +1 -0
- package/dist/nodes/model3d/generation.js +203 -0
- package/dist/nodes/model3d/generation.js.map +1 -0
- package/dist/nodes/model3d/glb.d.ts +29 -0
- package/dist/nodes/model3d/glb.d.ts.map +1 -0
- package/dist/nodes/model3d/glb.js +255 -0
- package/dist/nodes/model3d/glb.js.map +1 -0
- package/dist/nodes/model3d/index.d.ts +22 -0
- package/dist/nodes/model3d/index.d.ts.map +1 -0
- package/dist/nodes/model3d/index.js +41 -0
- package/dist/nodes/model3d/index.js.map +1 -0
- package/dist/nodes/model3d/io.d.ts +81 -0
- package/dist/nodes/model3d/io.d.ts.map +1 -0
- package/dist/nodes/model3d/io.js +162 -0
- package/dist/nodes/model3d/io.js.map +1 -0
- package/dist/nodes/model3d/mesh-ops.d.ts +40 -0
- package/dist/nodes/model3d/mesh-ops.d.ts.map +1 -0
- package/dist/nodes/model3d/mesh-ops.js +775 -0
- package/dist/nodes/model3d/mesh-ops.js.map +1 -0
- package/dist/nodes/model3d/nodes.d.ts +2 -0
- package/dist/nodes/model3d/nodes.d.ts.map +1 -0
- package/dist/nodes/model3d/nodes.js +2 -0
- package/dist/nodes/model3d/nodes.js.map +1 -0
- package/dist/nodes/model3d/ops.d.ts +44 -0
- package/dist/nodes/model3d/ops.d.ts.map +1 -0
- package/dist/nodes/model3d/ops.js +142 -0
- package/dist/nodes/model3d/ops.js.map +1 -0
- package/dist/nodes/model3d/render.d.ts +51 -0
- package/dist/nodes/model3d/render.d.ts.map +1 -0
- package/dist/nodes/model3d/render.js +124 -0
- package/dist/nodes/model3d/render.js.map +1 -0
- package/dist/nodes/model3d/render3d-core.d.ts +177 -0
- package/dist/nodes/model3d/render3d-core.d.ts.map +1 -0
- package/dist/nodes/model3d/render3d-core.js +366 -0
- package/dist/nodes/model3d/render3d-core.js.map +1 -0
- package/dist/nodes/model3d/render3d-headless.d.ts +75 -0
- package/dist/nodes/model3d/render3d-headless.d.ts.map +1 -0
- package/dist/nodes/model3d/render3d-headless.js +190 -0
- package/dist/nodes/model3d/render3d-headless.js.map +1 -0
- package/dist/nodes/model3d/render3d-page.d.ts +16 -0
- package/dist/nodes/model3d/render3d-page.d.ts.map +1 -0
- package/dist/nodes/model3d/render3d-page.js +51 -0
- package/dist/nodes/model3d/render3d-page.js.map +1 -0
- package/dist/nodes/model3d/transforms.d.ts +56 -0
- package/dist/nodes/model3d/transforms.d.ts.map +1 -0
- package/dist/nodes/model3d/transforms.js +148 -0
- package/dist/nodes/model3d/transforms.js.map +1 -0
- package/dist/nodes/model3d/types.d.ts +82 -0
- package/dist/nodes/model3d/types.d.ts.map +1 -0
- package/dist/nodes/model3d/types.js +2 -0
- package/dist/nodes/model3d/types.js.map +1 -0
- package/dist/nodes/model3d/utils.d.ts +46 -0
- package/dist/nodes/model3d/utils.d.ts.map +1 -0
- package/dist/nodes/model3d/utils.js +190 -0
- package/dist/nodes/model3d/utils.js.map +1 -0
- package/dist/nodes/model3d.d.ts +2 -0
- package/dist/nodes/model3d.d.ts.map +1 -0
- package/dist/nodes/model3d.js +2 -0
- package/dist/nodes/model3d.js.map +1 -0
- package/dist/nodes/placeholder-values.d.ts +10 -0
- package/dist/nodes/placeholder-values.d.ts.map +1 -0
- package/dist/nodes/placeholder-values.js +19 -0
- package/dist/nodes/placeholder-values.js.map +1 -0
- package/dist/nodes/script.d.ts +161 -0
- package/dist/nodes/script.d.ts.map +1 -0
- package/dist/nodes/script.js +673 -0
- package/dist/nodes/script.js.map +1 -0
- package/dist/nodes/storyboard.d.ts +269 -0
- package/dist/nodes/storyboard.d.ts.map +1 -0
- package/dist/nodes/storyboard.js +1017 -0
- package/dist/nodes/storyboard.js.map +1 -0
- package/dist/nodes/timeline/assetFiles.d.ts +19 -0
- package/dist/nodes/timeline/assetFiles.d.ts.map +1 -0
- package/dist/nodes/timeline/assetFiles.js +52 -0
- package/dist/nodes/timeline/assetFiles.js.map +1 -0
- package/dist/nodes/timeline/compositeRender.d.ts +60 -0
- package/dist/nodes/timeline/compositeRender.d.ts.map +1 -0
- package/dist/nodes/timeline/compositeRender.js +508 -0
- package/dist/nodes/timeline/compositeRender.js.map +1 -0
- package/dist/nodes/timeline/contactSheet.d.ts +43 -0
- package/dist/nodes/timeline/contactSheet.d.ts.map +1 -0
- package/dist/nodes/timeline/contactSheet.js +78 -0
- package/dist/nodes/timeline/contactSheet.js.map +1 -0
- package/dist/nodes/timeline/outputFormats.d.ts +69 -0
- package/dist/nodes/timeline/outputFormats.d.ts.map +1 -0
- package/dist/nodes/timeline/outputFormats.js +173 -0
- package/dist/nodes/timeline/outputFormats.js.map +1 -0
- package/dist/nodes/timeline/pngSequence.d.ts +44 -0
- package/dist/nodes/timeline/pngSequence.d.ts.map +1 -0
- package/dist/nodes/timeline/pngSequence.js +172 -0
- package/dist/nodes/timeline/pngSequence.js.map +1 -0
- package/dist/nodes/timeline/rasterizers.d.ts +47 -0
- package/dist/nodes/timeline/rasterizers.d.ts.map +1 -0
- package/dist/nodes/timeline/rasterizers.js +103 -0
- package/dist/nodes/timeline/rasterizers.js.map +1 -0
- package/dist/nodes/timeline/rawFrames.d.ts +149 -0
- package/dist/nodes/timeline/rawFrames.d.ts.map +1 -0
- package/dist/nodes/timeline/rawFrames.js +598 -0
- package/dist/nodes/timeline/rawFrames.js.map +1 -0
- package/dist/nodes/timeline.d.ts +206 -0
- package/dist/nodes/timeline.d.ts.map +1 -0
- package/dist/nodes/timeline.js +1395 -0
- package/dist/nodes/timeline.js.map +1 -0
- package/dist/nodes/video.d.ts +705 -0
- package/dist/nodes/video.d.ts.map +1 -0
- package/dist/nodes/video.js +2672 -0
- package/dist/nodes/video.js.map +1 -0
- package/dist/render3d-page.js +4118 -0
- package/package.json +95 -0
|
@@ -0,0 +1,1395 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Timeline nodes — operate on persisted timeline sequences (the video
|
|
3
|
+
* editor's documents) referenced by the `timeline` type.
|
|
4
|
+
*
|
|
5
|
+
* RenderTimeline composites the sequence frame by frame through the same GPU
|
|
6
|
+
* compositor and scene model the editor's preview and its in-browser export
|
|
7
|
+
* use (`@nodetool-ai/timeline/render`), so a workflow render is the picture
|
|
8
|
+
* the user previewed: clip placement, transforms, opacity, blend modes,
|
|
9
|
+
* transitions, animations, effects, captions, text and shapes, across every
|
|
10
|
+
* visual track. ffmpeg decodes clip media into RGBA and encodes the result.
|
|
11
|
+
*
|
|
12
|
+
* Compositing needs a WebGPU device, which means the optional `webgpu` (Dawn)
|
|
13
|
+
* package and a working driver. Both profiles ship Dawn (D9,
|
|
14
|
+
* docs/plans/motion-graphics.md), and the Docker image installs lavapipe as its
|
|
15
|
+
* Vulkan driver, so a server render is composited too. A host that still has no
|
|
16
|
+
* adapter falls back to the older ffmpeg rough cut — video/image clips
|
|
17
|
+
* normalized and concatenated in start order — which ignores everything above.
|
|
18
|
+
* The fallback is a `warning` on the job and `metadata.render_mode` on the
|
|
19
|
+
* output says which path ran: "composited" or "rough_cut".
|
|
20
|
+
*
|
|
21
|
+
* `format` picks the container and `alpha` decides whether the frames keep a
|
|
22
|
+
* transparency channel: `webm` (VP9, `yuva420p`), `mov` (ProRes 4444,
|
|
23
|
+
* `yuva444p10le`) and `png_sequence` (a zip of PNGs plus a `manifest.json`)
|
|
24
|
+
* carry it, `mp4` does not and refuses the pair. The table is
|
|
25
|
+
* `timeline/outputFormats.ts`.
|
|
26
|
+
*/
|
|
27
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
28
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
29
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
30
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
31
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
32
|
+
};
|
|
33
|
+
import { BaseNode, prop } from "@nodetool-ai/node-sdk";
|
|
34
|
+
import { loadMediaRefBytes } from "@nodetool-ai/runtime";
|
|
35
|
+
import { createTimeOrderedUuid, DEFAULT_MIDI_INSTRUMENT, encodeWavPcm16, ffmpegFadeCurve, fillTimelineText, resolveClipFades, retargetSequence, hasTimeRemap, sourceRate, makeClip, makeSequence, makeTrack, renderMidiClip, resolveTempo, timeRemapAudioSegments } from "@nodetool-ai/timeline";
|
|
36
|
+
import { tagAsNode, tagAsServer } from "@nodetool-ai/nodes-utils";
|
|
37
|
+
import { promises as fs } from "node:fs";
|
|
38
|
+
import os from "node:os";
|
|
39
|
+
import path from "node:path";
|
|
40
|
+
import { AssetFiles } from "./timeline/assetFiles.js";
|
|
41
|
+
import { execFfmpeg, execFfprobe, ffprobeDuration, FFMPEG_MAX_BUFFER, MissingBinaryError } from "./ffmpeg-helpers.js";
|
|
42
|
+
import { CompositorUnavailableError, renderTimelineComposited } from "./timeline/compositeRender.js";
|
|
43
|
+
import { ALPHA_OUTPUT_FORMATS, MissingEncoderError, resolveTimelineOutput, TIMELINE_OUTPUT_FORMATS } from "./timeline/outputFormats.js";
|
|
44
|
+
import { ffmpegHasEncoder } from "./timeline/rawFrames.js";
|
|
45
|
+
import { stringValues } from "./placeholder-values.js";
|
|
46
|
+
import { DEFAULT_SHUTTER_ANGLE, MAX_MOTION_BLUR_SAMPLES } from "@nodetool-ai/timeline/render";
|
|
47
|
+
import { isNonEmptyString, isNumber, isPositiveNumber, isString } from "@nodetool-ai/node-sdk";
|
|
48
|
+
async function loadTimelineSequence(ref, context) {
|
|
49
|
+
const timelineRef = (ref ?? {});
|
|
50
|
+
if (!timelineRef.id) {
|
|
51
|
+
throw new Error("Timeline input is empty — connect a Constant Timeline node and pick a timeline");
|
|
52
|
+
}
|
|
53
|
+
if (!context) {
|
|
54
|
+
throw new Error("Timeline nodes require a processing context");
|
|
55
|
+
}
|
|
56
|
+
const seq = (await context.getTimelineSequence(timelineRef.id));
|
|
57
|
+
if (!seq) {
|
|
58
|
+
throw new Error(`Timeline sequence not found: ${timelineRef.id}`);
|
|
59
|
+
}
|
|
60
|
+
return seq;
|
|
61
|
+
}
|
|
62
|
+
async function ffprobeHasAudio(filePath) {
|
|
63
|
+
try {
|
|
64
|
+
const { stdout } = await execFfprobe([
|
|
65
|
+
"-v",
|
|
66
|
+
"error",
|
|
67
|
+
"-select_streams",
|
|
68
|
+
"a",
|
|
69
|
+
"-show_entries",
|
|
70
|
+
"stream=codec_type",
|
|
71
|
+
"-of",
|
|
72
|
+
"default=noprint_wrappers=1:nokey=1",
|
|
73
|
+
filePath
|
|
74
|
+
]);
|
|
75
|
+
return stdout.trim().length > 0;
|
|
76
|
+
}
|
|
77
|
+
catch (error) {
|
|
78
|
+
if (error instanceof MissingBinaryError)
|
|
79
|
+
throw error;
|
|
80
|
+
return false;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
function trackById(tracks) {
|
|
84
|
+
return new Map(tracks.map((t) => [t.id, t]));
|
|
85
|
+
}
|
|
86
|
+
/** Clips that take part in the rough cut, in render order. */
|
|
87
|
+
function renderableVideoClips(seq) {
|
|
88
|
+
const tracks = trackById(seq.tracks);
|
|
89
|
+
return seq.clips
|
|
90
|
+
.filter((clip) => {
|
|
91
|
+
const track = tracks.get(clip.trackId);
|
|
92
|
+
return (track?.type === "video" &&
|
|
93
|
+
track.visible !== false &&
|
|
94
|
+
!clip.hidden &&
|
|
95
|
+
(clip.mediaType === "video" || clip.mediaType === "image") &&
|
|
96
|
+
!!clip.currentAssetId &&
|
|
97
|
+
clip.durationMs > 0);
|
|
98
|
+
})
|
|
99
|
+
.sort((a, b) => a.startMs - b.startMs);
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Clips the mix draws from: audio-track clips backed by an asset, and midi
|
|
103
|
+
* clips, whose sound is rendered from the notes they carry rather than fetched
|
|
104
|
+
* (they have no `currentAssetId` at all).
|
|
105
|
+
*/
|
|
106
|
+
function mixableAudioClips(seq) {
|
|
107
|
+
const tracks = trackById(seq.tracks);
|
|
108
|
+
return seq.clips
|
|
109
|
+
.filter((clip) => {
|
|
110
|
+
const track = tracks.get(clip.trackId);
|
|
111
|
+
if (!track || track.muted === true || clip.muted || clip.hidden) {
|
|
112
|
+
return false;
|
|
113
|
+
}
|
|
114
|
+
if (clip.durationMs <= 0)
|
|
115
|
+
return false;
|
|
116
|
+
if (track.type === "midi") {
|
|
117
|
+
return clip.mediaType === "midi" && (clip.notes?.length ?? 0) > 0;
|
|
118
|
+
}
|
|
119
|
+
return (track.type === "audio" &&
|
|
120
|
+
clip.mediaType === "audio" &&
|
|
121
|
+
!!clip.currentAssetId);
|
|
122
|
+
})
|
|
123
|
+
.sort((a, b) => a.startMs - b.startMs);
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* linkIds of audio-track clips that are the detached audio representation of a
|
|
127
|
+
* video clip. The mere presence of such a clip means the linked video's own
|
|
128
|
+
* muxed audio must be suppressed (the editor mutes the video element; audio
|
|
129
|
+
* comes only from audio-track clips). This is intentionally NOT gated on
|
|
130
|
+
* mute/hidden — those govern whether the audio clip is mixed in, not whether
|
|
131
|
+
* the video's embedded audio resurfaces. If the user deletes the audio clip,
|
|
132
|
+
* the store auto-unlinks the lone survivor, clearing the video's linkId, so the
|
|
133
|
+
* video's audio returns on its own.
|
|
134
|
+
*/
|
|
135
|
+
export function extractedAudioLinkIds(seq) {
|
|
136
|
+
const tracks = trackById(seq.tracks);
|
|
137
|
+
const ids = new Set();
|
|
138
|
+
for (const clip of seq.clips) {
|
|
139
|
+
const track = tracks.get(clip.trackId);
|
|
140
|
+
if (track?.type === "audio" &&
|
|
141
|
+
clip.mediaType === "audio" &&
|
|
142
|
+
isNonEmptyString(clip.linkId)) {
|
|
143
|
+
ids.add(clip.linkId);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return ids;
|
|
147
|
+
}
|
|
148
|
+
/** Scale + letterbox to the sequence frame, normalize fps and pixel format. */
|
|
149
|
+
function segmentVideoFilter(width, height, fps) {
|
|
150
|
+
return (`scale=${width}:${height}:force_original_aspect_ratio=decrease,` +
|
|
151
|
+
`pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black,` +
|
|
152
|
+
`fps=${fps},format=yuv420p`);
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Encode one clip into a uniform segment (same codec, resolution, fps,
|
|
156
|
+
* stereo AAC audio) so segments can be concatenated losslessly.
|
|
157
|
+
*/
|
|
158
|
+
async function encodeSegment(opts) {
|
|
159
|
+
const { clip, srcPath, segPath, width, height, fps, suppressEmbeddedAudio } = opts;
|
|
160
|
+
const durationS = clip.durationMs / 1000;
|
|
161
|
+
const filter = segmentVideoFilter(width, height, fps);
|
|
162
|
+
const silentAudio = [
|
|
163
|
+
"-f",
|
|
164
|
+
"lavfi",
|
|
165
|
+
"-t",
|
|
166
|
+
String(durationS),
|
|
167
|
+
"-i",
|
|
168
|
+
"anullsrc=channel_layout=stereo:sample_rate=48000"
|
|
169
|
+
];
|
|
170
|
+
if (clip.mediaType === "image") {
|
|
171
|
+
await execFfmpeg([
|
|
172
|
+
"-y",
|
|
173
|
+
"-loop",
|
|
174
|
+
"1",
|
|
175
|
+
"-t",
|
|
176
|
+
String(durationS),
|
|
177
|
+
"-i",
|
|
178
|
+
srcPath,
|
|
179
|
+
...silentAudio,
|
|
180
|
+
"-filter_complex",
|
|
181
|
+
`[0:v]${filter}[v]`,
|
|
182
|
+
"-map",
|
|
183
|
+
"[v]",
|
|
184
|
+
"-map",
|
|
185
|
+
"1:a",
|
|
186
|
+
"-c:v",
|
|
187
|
+
"libx264",
|
|
188
|
+
"-preset",
|
|
189
|
+
"veryfast",
|
|
190
|
+
"-c:a",
|
|
191
|
+
"aac",
|
|
192
|
+
"-shortest",
|
|
193
|
+
segPath
|
|
194
|
+
], { maxBuffer: FFMPEG_MAX_BUFFER });
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
const hasAudio = suppressEmbeddedAudio
|
|
198
|
+
? false
|
|
199
|
+
: await ffprobeHasAudio(srcPath);
|
|
200
|
+
const seek = [];
|
|
201
|
+
if (isPositiveNumber(clip.inPointMs)) {
|
|
202
|
+
seek.push("-ss", String(clip.inPointMs / 1000));
|
|
203
|
+
}
|
|
204
|
+
if (isPositiveNumber(clip.outPointMs)) {
|
|
205
|
+
seek.push("-to", String(clip.outPointMs / 1000));
|
|
206
|
+
}
|
|
207
|
+
await execFfmpeg([
|
|
208
|
+
"-y",
|
|
209
|
+
...seek,
|
|
210
|
+
"-i",
|
|
211
|
+
srcPath,
|
|
212
|
+
...(hasAudio ? [] : silentAudio),
|
|
213
|
+
"-filter_complex",
|
|
214
|
+
`[0:v]${filter}[v]`,
|
|
215
|
+
"-map",
|
|
216
|
+
"[v]",
|
|
217
|
+
"-map",
|
|
218
|
+
hasAudio ? "0:a" : "1:a",
|
|
219
|
+
"-t",
|
|
220
|
+
String(durationS),
|
|
221
|
+
"-c:v",
|
|
222
|
+
"libx264",
|
|
223
|
+
"-preset",
|
|
224
|
+
"veryfast",
|
|
225
|
+
"-c:a",
|
|
226
|
+
"aac",
|
|
227
|
+
"-ar",
|
|
228
|
+
"48000",
|
|
229
|
+
"-ac",
|
|
230
|
+
"2",
|
|
231
|
+
segPath
|
|
232
|
+
], { maxBuffer: FFMPEG_MAX_BUFFER });
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Clips whose embedded audio the composited path must mix in itself: the
|
|
236
|
+
* rough cut carried a video clip's audio through its segment, but a
|
|
237
|
+
* frame-by-frame composite has no audio at all. Muted clips/tracks and clips
|
|
238
|
+
* whose audio was extracted onto an audio track (see
|
|
239
|
+
* {@link extractedAudioLinkIds}) are left out.
|
|
240
|
+
*/
|
|
241
|
+
function embeddedAudioClips(seq) {
|
|
242
|
+
const tracks = trackById(seq.tracks);
|
|
243
|
+
const suppressed = extractedAudioLinkIds(seq);
|
|
244
|
+
return seq.clips
|
|
245
|
+
.filter((clip) => {
|
|
246
|
+
const track = tracks.get(clip.trackId);
|
|
247
|
+
return ((track?.type === "video" || track?.type === "overlay") &&
|
|
248
|
+
track.muted !== true &&
|
|
249
|
+
!clip.muted &&
|
|
250
|
+
clip.mediaType === "video" &&
|
|
251
|
+
!!clip.currentAssetId &&
|
|
252
|
+
clip.durationMs > 0 &&
|
|
253
|
+
!(isString(clip.linkId) && suppressed.has(clip.linkId)));
|
|
254
|
+
})
|
|
255
|
+
.sort((a, b) => a.startMs - b.startMs);
|
|
256
|
+
}
|
|
257
|
+
/** Seconds, at millisecond resolution, for an ffmpeg filter argument. */
|
|
258
|
+
function ffSeconds(ms) {
|
|
259
|
+
return (Math.max(0, ms) / 1000).toFixed(3);
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* `afade` steps for a clip's fades over a stream in which the clip begins at
|
|
263
|
+
* `clipStartInStreamMs`. Both edges read the curve the document authored, the
|
|
264
|
+
* same one the editor draws and the preview sounds: every `ClipFadeShape` maps
|
|
265
|
+
* to the `afade` curve with that gain function.
|
|
266
|
+
*
|
|
267
|
+
* Lengths come from `resolveClipFades`, so two fades longer together than the
|
|
268
|
+
* clip meet in its middle here exactly as they do on screen.
|
|
269
|
+
*/
|
|
270
|
+
function clipFadeSteps(clip, clipStartInStreamMs) {
|
|
271
|
+
const fades = resolveClipFades(clip, clip.durationMs);
|
|
272
|
+
const steps = [];
|
|
273
|
+
if (fades.fadeInMs > 0) {
|
|
274
|
+
steps.push(`afade=t=in:st=${ffSeconds(clipStartInStreamMs)}:d=${ffSeconds(fades.fadeInMs)}:curve=${ffmpegFadeCurve(fades.fadeInShape)}`);
|
|
275
|
+
}
|
|
276
|
+
if (fades.fadeOutMs > 0) {
|
|
277
|
+
const startMs = clipStartInStreamMs + clip.durationMs - fades.fadeOutMs;
|
|
278
|
+
steps.push(`afade=t=out:st=${ffSeconds(startMs)}:d=${ffSeconds(fades.fadeOutMs)}:curve=${ffmpegFadeCurve(fades.fadeOutShape)}`);
|
|
279
|
+
}
|
|
280
|
+
return steps;
|
|
281
|
+
}
|
|
282
|
+
/** Whether either edge of the clip carries a fade worth rendering. */
|
|
283
|
+
function hasClipFade(clip) {
|
|
284
|
+
const fades = resolveClipFades(clip, clip.durationMs);
|
|
285
|
+
return fades.fadeInMs > 0 || fades.fadeOutMs > 0;
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Per-clip audio chain: trim to in/out, apply gain, fade both edges, delay to
|
|
289
|
+
* timeline start. The fades precede the delay, so their times are measured
|
|
290
|
+
* from the clip's own start.
|
|
291
|
+
*/
|
|
292
|
+
function audioClipFilter(clip, inputIndex, label) {
|
|
293
|
+
const steps = [];
|
|
294
|
+
// A midi clip's WAV *is* its window: `renderMidiClip` already dropped
|
|
295
|
+
// everything before `inPointMs` and stopped at `durationMs`. Trimming by the
|
|
296
|
+
// source in/out again would cut the same window twice and lose the head of
|
|
297
|
+
// every trimmed clip.
|
|
298
|
+
const isMidi = clip.mediaType === "midi";
|
|
299
|
+
const inS = !isMidi && isNumber(clip.inPointMs) ? clip.inPointMs / 1000 : 0;
|
|
300
|
+
const outS = !isMidi && isPositiveNumber(clip.outPointMs)
|
|
301
|
+
? clip.outPointMs / 1000
|
|
302
|
+
: inS + clip.durationMs / 1000;
|
|
303
|
+
steps.push(`atrim=start=${inS}:end=${outS}`, "asetpts=PTS-STARTPTS");
|
|
304
|
+
if (isNumber(clip.volumeDb) && clip.volumeDb !== 0) {
|
|
305
|
+
steps.push(`volume=${clip.volumeDb}dB`);
|
|
306
|
+
}
|
|
307
|
+
steps.push(...clipFadeSteps(clip, 0));
|
|
308
|
+
const delay = Math.max(0, Math.round(clip.startMs));
|
|
309
|
+
steps.push(`adelay=${delay}|${delay}`);
|
|
310
|
+
return `[${inputIndex}:a]${steps.join(",")}[${label}]`;
|
|
311
|
+
}
|
|
312
|
+
/** `rate` as atempo factors, each inside the filter's own 0.5–2 range. */
|
|
313
|
+
function atempoChain(rate) {
|
|
314
|
+
const factors = [];
|
|
315
|
+
let remaining = rate;
|
|
316
|
+
while (remaining > 2) {
|
|
317
|
+
factors.push("2");
|
|
318
|
+
remaining /= 2;
|
|
319
|
+
}
|
|
320
|
+
while (remaining < 0.5) {
|
|
321
|
+
factors.push("0.5");
|
|
322
|
+
remaining *= 2;
|
|
323
|
+
}
|
|
324
|
+
factors.push(String(Number(remaining.toFixed(6))));
|
|
325
|
+
return factors;
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* One constant-rate stretch of a time-remapped clip: trim the source span the
|
|
329
|
+
* curve names, retime it to the stretch's timeline length, apply the clip's
|
|
330
|
+
* gain, and delay it to where the stretch starts.
|
|
331
|
+
*
|
|
332
|
+
* `atempo` retimes without moving the pitch and needs no knowledge of the
|
|
333
|
+
* input's sample rate, which `asetrate` would. The browser preview has no
|
|
334
|
+
* time-stretch to reach for — WebAudio offers only `playbackRate` — so a steep
|
|
335
|
+
* ramp is pitched there and is not here.
|
|
336
|
+
*/
|
|
337
|
+
function audioRemapSegmentFilter(clip, segment, inputIndex, label) {
|
|
338
|
+
const steps = [
|
|
339
|
+
`atrim=start=${segment.sourceStartMs / 1000}:end=${segment.sourceEndMs / 1000}`,
|
|
340
|
+
"asetpts=PTS-STARTPTS",
|
|
341
|
+
...atempoChain(segment.rate).map((f) => `atempo=${f}`)
|
|
342
|
+
];
|
|
343
|
+
if (isNumber(clip.volumeDb) && clip.volumeDb !== 0) {
|
|
344
|
+
steps.push(`volume=${clip.volumeDb}dB`);
|
|
345
|
+
}
|
|
346
|
+
const delay = Math.max(0, Math.round(segment.timelineStartMs));
|
|
347
|
+
steps.push(`adelay=${delay}|${delay}`);
|
|
348
|
+
return `[${inputIndex}:a]${steps.join(",")}[${label}]`;
|
|
349
|
+
}
|
|
350
|
+
/** Anything that draws: media, titles, shapes, or a caption riding a clip. */
|
|
351
|
+
function hasRenderableVisual(seq) {
|
|
352
|
+
const tracks = trackById(seq.tracks);
|
|
353
|
+
return seq.clips.some((clip) => {
|
|
354
|
+
const track = tracks.get(clip.trackId);
|
|
355
|
+
if (!track || track.visible === false || clip.hidden)
|
|
356
|
+
return false;
|
|
357
|
+
if (clip.caption)
|
|
358
|
+
return true;
|
|
359
|
+
if (track.type !== "video" && track.type !== "overlay")
|
|
360
|
+
return false;
|
|
361
|
+
if (clip.mediaType === "text")
|
|
362
|
+
return !!clip.textStyle;
|
|
363
|
+
if (clip.mediaType === "shape")
|
|
364
|
+
return !!clip.shapeStyle;
|
|
365
|
+
return ((clip.mediaType === "video" || clip.mediaType === "image") &&
|
|
366
|
+
!!clip.currentAssetId);
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
/** The sequence's own length, or the end of its last clip when it has none. */
|
|
370
|
+
function totalDurationMs(seq) {
|
|
371
|
+
if (seq.durationMs > 0)
|
|
372
|
+
return seq.durationMs;
|
|
373
|
+
return seq.clips.reduce((end, clip) => Math.max(end, clip.startMs + clip.durationMs), 0);
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* The file ffmpeg reads for each clip asset, resolved once per asset: the
|
|
377
|
+
* asset's own local file when it has one, else a copy in the work dir.
|
|
378
|
+
*/
|
|
379
|
+
/** Sample rate the midi renderer runs at, matching the mix's own `-ar`. */
|
|
380
|
+
const MIDI_SAMPLE_RATE = 48000;
|
|
381
|
+
/**
|
|
382
|
+
* Renders each midi clip's notes to a WAV on disk once, for ffmpeg to read.
|
|
383
|
+
*
|
|
384
|
+
* The buffer `renderMidiClip` returns is exactly the clip's window — it starts
|
|
385
|
+
* at the clip's timeline start and is `durationMs` long — so the file needs no
|
|
386
|
+
* source-side trim, only the `adelay` that puts it at its offset.
|
|
387
|
+
*/
|
|
388
|
+
class MidiFiles {
|
|
389
|
+
workDir;
|
|
390
|
+
files = new Map();
|
|
391
|
+
tracks;
|
|
392
|
+
bpm;
|
|
393
|
+
constructor(workDir, seq) {
|
|
394
|
+
this.workDir = workDir;
|
|
395
|
+
this.tracks = trackById(seq.tracks);
|
|
396
|
+
this.bpm = resolveTempo(seq).bpm;
|
|
397
|
+
}
|
|
398
|
+
path(clip) {
|
|
399
|
+
let pending = this.files.get(clip.id);
|
|
400
|
+
if (!pending) {
|
|
401
|
+
pending = this.write(clip);
|
|
402
|
+
this.files.set(clip.id, pending);
|
|
403
|
+
}
|
|
404
|
+
return pending;
|
|
405
|
+
}
|
|
406
|
+
async write(clip) {
|
|
407
|
+
// The instrument belongs to the track, not the clip.
|
|
408
|
+
const instrument = this.tracks.get(clip.trackId)?.instrument ?? DEFAULT_MIDI_INSTRUMENT;
|
|
409
|
+
const mono = renderMidiClip({
|
|
410
|
+
clip,
|
|
411
|
+
bpm: this.bpm,
|
|
412
|
+
instrument,
|
|
413
|
+
sampleRate: MIDI_SAMPLE_RATE
|
|
414
|
+
});
|
|
415
|
+
const file = path.join(this.workDir, `midi_${clip.id}.wav`);
|
|
416
|
+
await fs.writeFile(file, encodeWavPcm16(mono, MIDI_SAMPLE_RATE));
|
|
417
|
+
return file;
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
/**
|
|
421
|
+
* The file ffmpeg reads one audible clip from: the materialized asset, or the
|
|
422
|
+
* WAV rendered from a midi clip's notes.
|
|
423
|
+
*/
|
|
424
|
+
function audioSourcePath(clip, assets, midi) {
|
|
425
|
+
if (clip.mediaType === "midi")
|
|
426
|
+
return midi.path(clip);
|
|
427
|
+
return clip.currentAssetId
|
|
428
|
+
? assets.path(clip.currentAssetId)
|
|
429
|
+
: Promise.resolve(null);
|
|
430
|
+
}
|
|
431
|
+
/**
|
|
432
|
+
* The pre-compositor rough cut, kept as the fallback for hosts without a GPU:
|
|
433
|
+
* each video/image clip is normalized to the sequence frame and the segments
|
|
434
|
+
* are concatenated in start order. Gaps, overlaps, transforms and every other
|
|
435
|
+
* layer property are lost — it is a cut, not a render.
|
|
436
|
+
*/
|
|
437
|
+
async function renderRoughCut(opts) {
|
|
438
|
+
const { seq, clips, audioClips, assets, workDir, width, height, fps } = opts;
|
|
439
|
+
const suppressedLinkIds = extractedAudioLinkIds(seq);
|
|
440
|
+
const segments = [];
|
|
441
|
+
for (const [i, clip] of clips.entries()) {
|
|
442
|
+
const srcPath = clip.currentAssetId
|
|
443
|
+
? await assets.path(clip.currentAssetId)
|
|
444
|
+
: null;
|
|
445
|
+
if (!srcPath) {
|
|
446
|
+
console.warn(`RenderTimeline: skipping clip "${clip.name}" — asset ${clip.currentAssetId} not found`);
|
|
447
|
+
continue;
|
|
448
|
+
}
|
|
449
|
+
const segPath = path.join(workDir, `seg_${i}.mp4`);
|
|
450
|
+
await encodeSegment({
|
|
451
|
+
clip,
|
|
452
|
+
srcPath,
|
|
453
|
+
segPath,
|
|
454
|
+
width,
|
|
455
|
+
height,
|
|
456
|
+
fps,
|
|
457
|
+
suppressEmbeddedAudio: isString(clip.linkId) && suppressedLinkIds.has(clip.linkId)
|
|
458
|
+
});
|
|
459
|
+
segments.push(segPath);
|
|
460
|
+
}
|
|
461
|
+
const basePath = path.join(workDir, "base.mp4");
|
|
462
|
+
if (segments.length > 0) {
|
|
463
|
+
const listPath = path.join(workDir, "segments.txt");
|
|
464
|
+
await fs.writeFile(listPath, segments.map((p) => `file '${p}'`).join("\n"));
|
|
465
|
+
await execFfmpeg(["-y", "-f", "concat", "-safe", "0", "-i", listPath, "-c", "copy", basePath], { maxBuffer: FFMPEG_MAX_BUFFER });
|
|
466
|
+
return basePath;
|
|
467
|
+
}
|
|
468
|
+
// Audio-only cut: synthesize a black picture to carry the mix.
|
|
469
|
+
//
|
|
470
|
+
// `Math.max()` with no arguments is -Infinity, and spreading an empty array
|
|
471
|
+
// is exactly that call. A timeline holding neither video nor audio reached
|
|
472
|
+
// ffmpeg as `-t -Infinity` and died on "Invalid duration for option t",
|
|
473
|
+
// naming neither the timeline nor the absent clips. `totalDurationMs` above
|
|
474
|
+
// seeds a `reduce` for the same computation and is unaffected — one file,
|
|
475
|
+
// two idioms, one of them wrong. Nothing renders from an empty timeline, so
|
|
476
|
+
// say so here rather than build a command that cannot parse.
|
|
477
|
+
if (audioClips.length === 0) {
|
|
478
|
+
throw new Error("Timeline has no clips to render. Add at least one video, image or " +
|
|
479
|
+
"audio clip before rendering.");
|
|
480
|
+
}
|
|
481
|
+
const totalS = audioClips.reduce((end, c) => Math.max(end, c.startMs + c.durationMs), 0) /
|
|
482
|
+
1000;
|
|
483
|
+
await execFfmpeg([
|
|
484
|
+
"-y",
|
|
485
|
+
"-f",
|
|
486
|
+
"lavfi",
|
|
487
|
+
"-t",
|
|
488
|
+
String(totalS),
|
|
489
|
+
"-i",
|
|
490
|
+
`color=black:s=${width}x${height}:r=${fps}`,
|
|
491
|
+
"-f",
|
|
492
|
+
"lavfi",
|
|
493
|
+
"-t",
|
|
494
|
+
String(totalS),
|
|
495
|
+
"-i",
|
|
496
|
+
"anullsrc=channel_layout=stereo:sample_rate=48000",
|
|
497
|
+
"-c:v",
|
|
498
|
+
"libx264",
|
|
499
|
+
"-preset",
|
|
500
|
+
"veryfast",
|
|
501
|
+
"-c:a",
|
|
502
|
+
"aac",
|
|
503
|
+
basePath
|
|
504
|
+
], { maxBuffer: FFMPEG_MAX_BUFFER });
|
|
505
|
+
return basePath;
|
|
506
|
+
}
|
|
507
|
+
/**
|
|
508
|
+
* Mix `clips` into `basePath` at their timeline offsets. `baseHasAudio` says
|
|
509
|
+
* whether the base video carries a soundtrack of its own to keep (the rough
|
|
510
|
+
* cut does; a composited render does not).
|
|
511
|
+
*
|
|
512
|
+
* The mux copies the video stream, so the container has to be the one the base
|
|
513
|
+
* was encoded into and the audio codec has to be legal inside it — AAC is not a
|
|
514
|
+
* WebM track, which is why `audioCodec` is passed rather than assumed.
|
|
515
|
+
*/
|
|
516
|
+
async function mixAudioInto(opts) {
|
|
517
|
+
const { basePath, clips, baseHasAudio, assets, midi, workDir, extension, audioCodec } = opts;
|
|
518
|
+
const inputs = ["-i", basePath];
|
|
519
|
+
const filters = [];
|
|
520
|
+
const labels = [];
|
|
521
|
+
let inputIndex = 1;
|
|
522
|
+
for (const [i, clip] of clips.entries()) {
|
|
523
|
+
const audioPath = await audioSourcePath(clip, assets, midi);
|
|
524
|
+
if (!audioPath) {
|
|
525
|
+
console.warn(`RenderTimeline: skipping audio of clip "${clip.name}" — asset ${clip.currentAssetId} not found`);
|
|
526
|
+
continue;
|
|
527
|
+
}
|
|
528
|
+
if (!baseHasAudio && !(await ffprobeHasAudio(audioPath)))
|
|
529
|
+
continue;
|
|
530
|
+
// Only a clip that plays at source rate can use the plain chain: a
|
|
531
|
+
// speedMultiplier is a rate too, and without atempo (and the wider source
|
|
532
|
+
// window it needs) a 2x clip's audio ran on past the clip.
|
|
533
|
+
if (!hasTimeRemap(clip) && sourceRate(clip) === 1) {
|
|
534
|
+
inputs.push("-i", audioPath);
|
|
535
|
+
const label = `a${i}`;
|
|
536
|
+
filters.push(audioClipFilter(clip, inputIndex, label));
|
|
537
|
+
labels.push(`[${label}]`);
|
|
538
|
+
inputIndex += 1;
|
|
539
|
+
continue;
|
|
540
|
+
}
|
|
541
|
+
// A remapped clip is not one span at one rate: the curve becomes
|
|
542
|
+
// constant-rate stretches and each is decoded, retimed and delayed on its
|
|
543
|
+
// own. A stretch the curve runs backwards over — or holds still on — is
|
|
544
|
+
// silent, as it is in the preview.
|
|
545
|
+
const segmentLabels = [];
|
|
546
|
+
for (const [j, segment] of timeRemapAudioSegments(clip).entries()) {
|
|
547
|
+
if (segment.reverse || !Number.isFinite(segment.rate))
|
|
548
|
+
continue;
|
|
549
|
+
inputs.push("-i", audioPath);
|
|
550
|
+
const label = `a${i}_${j}`;
|
|
551
|
+
filters.push(audioRemapSegmentFilter(clip, segment, inputIndex, label));
|
|
552
|
+
segmentLabels.push(`[${label}]`);
|
|
553
|
+
inputIndex += 1;
|
|
554
|
+
}
|
|
555
|
+
// A fade spans the clip, not one stretch of it, and a stretch that covers
|
|
556
|
+
// only part of the ramp cannot carry it — `afade` always runs a full 0→1.
|
|
557
|
+
// So a faded remapped clip is mixed back into one stream first, at its
|
|
558
|
+
// timeline offsets, and faded there. `normalize=0` keeps the level: the
|
|
559
|
+
// stretches abut rather than overlap, so nothing is summed.
|
|
560
|
+
if (segmentLabels.length > 0 && hasClipFade(clip)) {
|
|
561
|
+
const faded = `a${i}_fade`;
|
|
562
|
+
const steps = segmentLabels.length > 1
|
|
563
|
+
? [
|
|
564
|
+
`amix=inputs=${segmentLabels.length}:duration=longest:normalize=0`,
|
|
565
|
+
...clipFadeSteps(clip, clip.startMs)
|
|
566
|
+
]
|
|
567
|
+
: clipFadeSteps(clip, clip.startMs);
|
|
568
|
+
filters.push(`${segmentLabels.join("")}${steps.join(",")}[${faded}]`);
|
|
569
|
+
labels.push(`[${faded}]`);
|
|
570
|
+
continue;
|
|
571
|
+
}
|
|
572
|
+
labels.push(...segmentLabels);
|
|
573
|
+
}
|
|
574
|
+
if (labels.length === 0)
|
|
575
|
+
return basePath;
|
|
576
|
+
const sources = baseHasAudio ? [`[0:a]`, ...labels] : labels;
|
|
577
|
+
const mix = sources.length === 1
|
|
578
|
+
? `${sources[0]}apad[aout]`
|
|
579
|
+
: `${sources.join("")}amix=inputs=${sources.length}:duration=longest:normalize=0,apad[aout]`;
|
|
580
|
+
const outPath = path.join(workDir, `mixed.${extension}`);
|
|
581
|
+
await execFfmpeg([
|
|
582
|
+
"-y",
|
|
583
|
+
...inputs,
|
|
584
|
+
"-filter_complex",
|
|
585
|
+
[...filters, mix].join(";"),
|
|
586
|
+
"-map",
|
|
587
|
+
"0:v",
|
|
588
|
+
"-map",
|
|
589
|
+
"[aout]",
|
|
590
|
+
"-c:v",
|
|
591
|
+
"copy",
|
|
592
|
+
"-c:a",
|
|
593
|
+
audioCodec,
|
|
594
|
+
"-ar",
|
|
595
|
+
"48000",
|
|
596
|
+
"-ac",
|
|
597
|
+
"2",
|
|
598
|
+
// `apad` runs the mix past the picture; stop at the shorter stream so the
|
|
599
|
+
// render is exactly as long as the video.
|
|
600
|
+
"-shortest",
|
|
601
|
+
outPath
|
|
602
|
+
], { maxBuffer: FFMPEG_MAX_BUFFER });
|
|
603
|
+
return outPath;
|
|
604
|
+
}
|
|
605
|
+
/** Add the sequence's audible tracks and embedded video audio to a composited video. */
|
|
606
|
+
export async function mixCompositedTimelineAudio(opts) {
|
|
607
|
+
const { sequence, basePath, workDir, output, resolveAssetPath } = opts;
|
|
608
|
+
if (output.format === "png_sequence")
|
|
609
|
+
return basePath;
|
|
610
|
+
const clips = [...embeddedAudioClips(sequence), ...mixableAudioClips(sequence)];
|
|
611
|
+
if (clips.length === 0)
|
|
612
|
+
return basePath;
|
|
613
|
+
return mixAudioInto({
|
|
614
|
+
basePath,
|
|
615
|
+
clips,
|
|
616
|
+
baseHasAudio: false,
|
|
617
|
+
assets: { path: resolveAssetPath },
|
|
618
|
+
midi: new MidiFiles(workDir, sequence),
|
|
619
|
+
workDir,
|
|
620
|
+
extension: output.extension,
|
|
621
|
+
audioCodec: output.audioCodec ?? "aac"
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
/** Shortest gap between two `node_progress` posts — four a second. */
|
|
625
|
+
const PROGRESS_INTERVAL_MS = 250;
|
|
626
|
+
/** Smallest draft render. Below this a frame says nothing about the cut. */
|
|
627
|
+
const MIN_PREVIEW_SCALE = 0.1;
|
|
628
|
+
/**
|
|
629
|
+
* The fraction of the sequence size this render runs at.
|
|
630
|
+
*
|
|
631
|
+
* Unset, non-finite, and anything at or above 1 mean full size; anything
|
|
632
|
+
* smaller is clamped up to {@link MIN_PREVIEW_SCALE} rather than refused, so a
|
|
633
|
+
* draft is never rendered at a size nobody could read.
|
|
634
|
+
*/
|
|
635
|
+
function previewScaleOf(value) {
|
|
636
|
+
const scale = Number(value);
|
|
637
|
+
if (!Number.isFinite(scale) || scale >= 1)
|
|
638
|
+
return 1;
|
|
639
|
+
return Math.max(MIN_PREVIEW_SCALE, scale);
|
|
640
|
+
}
|
|
641
|
+
/** A scaled edge, rounded to an even number — h.264 encodes nothing else. */
|
|
642
|
+
function evenScaled(edge, scale) {
|
|
643
|
+
if (scale >= 1)
|
|
644
|
+
return edge;
|
|
645
|
+
return Math.max(2, Math.round((edge * scale) / 2) * 2);
|
|
646
|
+
}
|
|
647
|
+
/** An unset ref on the handle this render did not fill. */
|
|
648
|
+
function emptyVideoRef() {
|
|
649
|
+
return { type: "video", uri: "", data: null, asset_id: null };
|
|
650
|
+
}
|
|
651
|
+
function emptyFramesRef() {
|
|
652
|
+
return { type: "document", uri: "", asset_id: null, data: null };
|
|
653
|
+
}
|
|
654
|
+
/**
|
|
655
|
+
* Store a finished render in the host's asset store and describe it by id.
|
|
656
|
+
*
|
|
657
|
+
* A minute of 1080p is tens of megabytes; base64 of that sits at V8's string
|
|
658
|
+
* cap, so encoding the artifact into the ref fails after the whole render has
|
|
659
|
+
* already been paid for. With an asset store the bytes go there and the ref
|
|
660
|
+
* carries `asset_id` plus an `asset://` uri and no inline `data`. A host
|
|
661
|
+
* without one (the CLI, a unit test) keeps the inline encoding, which is the
|
|
662
|
+
* only way those callers see the result at all.
|
|
663
|
+
*/
|
|
664
|
+
async function storeRenderArtifact(args) {
|
|
665
|
+
const bytes = await fs.readFile(args.filePath);
|
|
666
|
+
if (typeof args.context.createAsset !== "function") {
|
|
667
|
+
return { asset_id: null, uri: "", data: bytes.toString("base64") };
|
|
668
|
+
}
|
|
669
|
+
const created = (await args.context.createAsset({
|
|
670
|
+
name: args.name,
|
|
671
|
+
contentType: args.contentType,
|
|
672
|
+
content: new Uint8Array(bytes)
|
|
673
|
+
}));
|
|
674
|
+
const assetId = created && typeof created.id === "string" && created.id.length > 0
|
|
675
|
+
? created.id
|
|
676
|
+
: null;
|
|
677
|
+
if (!assetId) {
|
|
678
|
+
return { asset_id: null, uri: "", data: bytes.toString("base64") };
|
|
679
|
+
}
|
|
680
|
+
return { asset_id: assetId, uri: `asset://${assetId}`, data: null };
|
|
681
|
+
}
|
|
682
|
+
export class RenderTimelineNode extends BaseNode {
|
|
683
|
+
static nodeType = "nodetool.timeline.RenderTimeline";
|
|
684
|
+
static title = "Render Timeline";
|
|
685
|
+
static description = "Render a timeline sequence to a video, composited exactly as the timeline editor previews it (tracks, transforms, transitions, effects, captions and text), with audio mixed in at each clip's offset. Writes MP4, WebM, MOV or a zipped PNG sequence, with an optional transparent background.\n timeline, render, video, export, cut, alpha\n\n Use cases:\n - Turn an edit assembled in the timeline editor into a shareable video\n - Export a title or lower third over transparency to composite elsewhere\n - Feed a rough cut into captioning, review, or upload nodes\n - Automate exports of timelines built by other workflow nodes";
|
|
686
|
+
static requiredRuntimes = ["ffmpeg"];
|
|
687
|
+
static metadataOutputTypes = {
|
|
688
|
+
output: "video",
|
|
689
|
+
frames: "document"
|
|
690
|
+
};
|
|
691
|
+
static inlineFields = ["timeline"];
|
|
692
|
+
static inputFields = ["timeline"];
|
|
693
|
+
async process(context) {
|
|
694
|
+
const seq = await loadTimelineSequence(this.timeline, context);
|
|
695
|
+
const ctx = context;
|
|
696
|
+
const scale = previewScaleOf(this.preview_scale);
|
|
697
|
+
const width = evenScaled(seq.width > 0 ? seq.width : 1920, scale);
|
|
698
|
+
const height = evenScaled(seq.height > 0 ? seq.height : 1080, scale);
|
|
699
|
+
const fps = seq.fps > 0 ? seq.fps : 30;
|
|
700
|
+
const includeAudio = this.include_audio !== false;
|
|
701
|
+
const audioClips = includeAudio ? mixableAudioClips(seq) : [];
|
|
702
|
+
if (!hasRenderableVisual(seq) && audioClips.length === 0) {
|
|
703
|
+
throw new Error(`Timeline "${seq.name}" has no renderable clips with media — generate or import clip media first`);
|
|
704
|
+
}
|
|
705
|
+
const durationMs = totalDurationMs(seq);
|
|
706
|
+
if (durationMs <= 0) {
|
|
707
|
+
throw new Error(`Timeline "${seq.name}" has zero duration`);
|
|
708
|
+
}
|
|
709
|
+
// Resolved before anything is decoded: an impossible format/alpha pair, or
|
|
710
|
+
// an ffmpeg with no encoder for it, must fail in a second rather than after
|
|
711
|
+
// a thousand composited frames.
|
|
712
|
+
const output = resolveTimelineOutput({
|
|
713
|
+
format: this.format,
|
|
714
|
+
alpha: this.alpha === true,
|
|
715
|
+
videoCodec: this.video_codec,
|
|
716
|
+
bitrate: this.bitrate,
|
|
717
|
+
motionBlurSamples: this.motion_blur_samples,
|
|
718
|
+
shutterAngle: this.shutter_angle
|
|
719
|
+
});
|
|
720
|
+
await this.assertEncoderAvailable(output);
|
|
721
|
+
const workDir = await fs.mkdtemp(path.join(os.tmpdir(), "nodetool-timeline-"));
|
|
722
|
+
try {
|
|
723
|
+
const assets = new AssetFiles(workDir, ctx);
|
|
724
|
+
const midi = new MidiFiles(workDir, seq);
|
|
725
|
+
let basePath;
|
|
726
|
+
let baseHasAudio;
|
|
727
|
+
let audioToMix;
|
|
728
|
+
let renderMode;
|
|
729
|
+
try {
|
|
730
|
+
basePath = path.join(workDir, `composited.${output.extension}`);
|
|
731
|
+
const { skippedClips } = await renderTimelineComposited({
|
|
732
|
+
sequence: seq,
|
|
733
|
+
width,
|
|
734
|
+
height,
|
|
735
|
+
fps,
|
|
736
|
+
durationMs,
|
|
737
|
+
resolveAssetPath: (assetId) => assets.path(assetId),
|
|
738
|
+
outPath: basePath,
|
|
739
|
+
output,
|
|
740
|
+
onProgress: this.progressReporter(ctx),
|
|
741
|
+
signal: ctx.signal
|
|
742
|
+
});
|
|
743
|
+
for (const name of skippedClips) {
|
|
744
|
+
this.log(ctx, `Clip "${name}" was skipped — its media could not be decoded`, "warning");
|
|
745
|
+
}
|
|
746
|
+
renderMode = "composited";
|
|
747
|
+
// A composite carries no sound: every audible clip is mixed in below,
|
|
748
|
+
// including the audio muxed into video clips.
|
|
749
|
+
baseHasAudio = false;
|
|
750
|
+
audioToMix = includeAudio
|
|
751
|
+
? [...embeddedAudioClips(seq), ...audioClips]
|
|
752
|
+
: [];
|
|
753
|
+
}
|
|
754
|
+
catch (error) {
|
|
755
|
+
if (!(error instanceof CompositorUnavailableError))
|
|
756
|
+
throw error;
|
|
757
|
+
// The rough cut is H.264 in MP4 over opaque black — it concatenates
|
|
758
|
+
// normalized segments and never touches the compositor. Handing that
|
|
759
|
+
// back for an alpha or non-MP4 request would answer a different
|
|
760
|
+
// question than the one asked, so the request fails instead.
|
|
761
|
+
if (output.format !== "mp4" || output.alpha) {
|
|
762
|
+
throw new Error(`${error.message} The "${output.format}" format` +
|
|
763
|
+
`${output.alpha ? " with transparency" : ""} needs the ` +
|
|
764
|
+
"compositor; the rough-cut fallback only writes opaque MP4.");
|
|
765
|
+
}
|
|
766
|
+
this.log(ctx, `${error.message} Falling back to a rough cut — clip transforms, ` +
|
|
767
|
+
"effects, transitions, captions and overlay tracks are not applied.", "warning");
|
|
768
|
+
basePath = await renderRoughCut({
|
|
769
|
+
seq,
|
|
770
|
+
clips: renderableVideoClips(seq),
|
|
771
|
+
audioClips,
|
|
772
|
+
assets,
|
|
773
|
+
workDir,
|
|
774
|
+
width,
|
|
775
|
+
height,
|
|
776
|
+
fps
|
|
777
|
+
});
|
|
778
|
+
renderMode = "rough_cut";
|
|
779
|
+
baseHasAudio = true;
|
|
780
|
+
audioToMix = audioClips;
|
|
781
|
+
}
|
|
782
|
+
// A PNG sequence is a zip of stills: there is no stream to mux a
|
|
783
|
+
// soundtrack onto, so the audio clips are reported rather than dropped
|
|
784
|
+
// silently.
|
|
785
|
+
if (output.format === "png_sequence") {
|
|
786
|
+
if (audioToMix.length > 0) {
|
|
787
|
+
this.log(ctx, "A PNG sequence carries no audio; the timeline's audio clips were not mixed in.", "warning");
|
|
788
|
+
}
|
|
789
|
+
const stored = await storeRenderArtifact({
|
|
790
|
+
context: ctx,
|
|
791
|
+
filePath: basePath,
|
|
792
|
+
name: `${seq.name || "timeline"}-frames.zip`,
|
|
793
|
+
contentType: output.mimeType
|
|
794
|
+
});
|
|
795
|
+
return {
|
|
796
|
+
output: emptyVideoRef(),
|
|
797
|
+
frames: {
|
|
798
|
+
type: "document",
|
|
799
|
+
uri: stored.uri,
|
|
800
|
+
asset_id: stored.asset_id,
|
|
801
|
+
data: stored.data,
|
|
802
|
+
metadata: {
|
|
803
|
+
render_mode: renderMode,
|
|
804
|
+
format: output.format,
|
|
805
|
+
alpha: output.alpha,
|
|
806
|
+
content_type: output.mimeType
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
};
|
|
810
|
+
}
|
|
811
|
+
const outPath = audioToMix.length > 0
|
|
812
|
+
? await mixAudioInto({
|
|
813
|
+
basePath,
|
|
814
|
+
clips: audioToMix,
|
|
815
|
+
baseHasAudio,
|
|
816
|
+
assets,
|
|
817
|
+
midi,
|
|
818
|
+
workDir,
|
|
819
|
+
extension: output.extension,
|
|
820
|
+
audioCodec: output.audioCodec ?? "aac"
|
|
821
|
+
})
|
|
822
|
+
: basePath;
|
|
823
|
+
const duration = await ffprobeDuration(outPath);
|
|
824
|
+
const stored = await storeRenderArtifact({
|
|
825
|
+
context: ctx,
|
|
826
|
+
filePath: outPath,
|
|
827
|
+
name: `${seq.name || "timeline"}.${output.extension}`,
|
|
828
|
+
contentType: output.mimeType
|
|
829
|
+
});
|
|
830
|
+
return {
|
|
831
|
+
output: {
|
|
832
|
+
type: "video",
|
|
833
|
+
asset_id: stored.asset_id,
|
|
834
|
+
uri: stored.uri,
|
|
835
|
+
data: stored.data,
|
|
836
|
+
format: output.extension,
|
|
837
|
+
duration: duration > 0 ? duration : null,
|
|
838
|
+
// Which of the two paths produced these bytes. The docker smoke test
|
|
839
|
+
// asserts "composited": it is what proves the image renders the
|
|
840
|
+
// picture the editor previews rather than a concatenation.
|
|
841
|
+
metadata: {
|
|
842
|
+
render_mode: renderMode,
|
|
843
|
+
format: output.format,
|
|
844
|
+
alpha: output.alpha
|
|
845
|
+
}
|
|
846
|
+
},
|
|
847
|
+
frames: emptyFramesRef()
|
|
848
|
+
};
|
|
849
|
+
}
|
|
850
|
+
finally {
|
|
851
|
+
await fs.rm(workDir, { recursive: true, force: true });
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* Refuse a format this ffmpeg cannot encode, by name.
|
|
856
|
+
*
|
|
857
|
+
* ProRes and VP9 are both optional at ffmpeg build time. Without this the
|
|
858
|
+
* render composites every frame first and then dies on ffmpeg's own
|
|
859
|
+
* "Unknown encoder" line, which says nothing about what to pick instead.
|
|
860
|
+
*/
|
|
861
|
+
async assertEncoderAvailable(output) {
|
|
862
|
+
const encoder = output.requiredEncoder;
|
|
863
|
+
// MP4/H.264 is the path every existing render already took, and probing it
|
|
864
|
+
// would turn "ffmpeg is not installed" into a message about an encoder.
|
|
865
|
+
if (!encoder || output.format === "mp4")
|
|
866
|
+
return;
|
|
867
|
+
if (await ffmpegHasEncoder(encoder))
|
|
868
|
+
return;
|
|
869
|
+
throw new MissingEncoderError(encoder, output.format);
|
|
870
|
+
}
|
|
871
|
+
/** Post on the node's log channel, where the run has one. */
|
|
872
|
+
log(context, content, severity) {
|
|
873
|
+
if (!this.__node_id)
|
|
874
|
+
return;
|
|
875
|
+
context.postMessage({
|
|
876
|
+
type: "log_update",
|
|
877
|
+
node_id: this.__node_id,
|
|
878
|
+
node_name: RenderTimelineNode.title,
|
|
879
|
+
content,
|
|
880
|
+
severity,
|
|
881
|
+
workflow_id: context.workflowId
|
|
882
|
+
});
|
|
883
|
+
}
|
|
884
|
+
/**
|
|
885
|
+
* A `node_progress` reporter for the frame loop, rate-limited to four posts a
|
|
886
|
+
* second. A minute of 1080p is thousands of frames and every post crosses the
|
|
887
|
+
* websocket; the final frame always posts, so the bar reaches its end.
|
|
888
|
+
*/
|
|
889
|
+
progressReporter(context) {
|
|
890
|
+
const nodeId = this.__node_id;
|
|
891
|
+
let lastPostMs = 0;
|
|
892
|
+
return (frame, totalFrames) => {
|
|
893
|
+
if (!nodeId)
|
|
894
|
+
return;
|
|
895
|
+
const now = Date.now();
|
|
896
|
+
if (frame < totalFrames && now - lastPostMs < PROGRESS_INTERVAL_MS) {
|
|
897
|
+
return;
|
|
898
|
+
}
|
|
899
|
+
lastPostMs = now;
|
|
900
|
+
context.postMessage({
|
|
901
|
+
type: "node_progress",
|
|
902
|
+
node_id: nodeId,
|
|
903
|
+
progress: frame,
|
|
904
|
+
total: totalFrames,
|
|
905
|
+
workflow_id: context.workflowId
|
|
906
|
+
});
|
|
907
|
+
};
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
__decorate([
|
|
911
|
+
prop({
|
|
912
|
+
type: "timeline",
|
|
913
|
+
default: { type: "timeline", id: null, data: null },
|
|
914
|
+
title: "Timeline",
|
|
915
|
+
description: "The timeline sequence to render."
|
|
916
|
+
})
|
|
917
|
+
], RenderTimelineNode.prototype, "timeline", void 0);
|
|
918
|
+
__decorate([
|
|
919
|
+
prop({
|
|
920
|
+
type: "bool",
|
|
921
|
+
default: true,
|
|
922
|
+
title: "Include audio",
|
|
923
|
+
description: "Mix audio-track clips into the rendered video."
|
|
924
|
+
})
|
|
925
|
+
], RenderTimelineNode.prototype, "include_audio", void 0);
|
|
926
|
+
__decorate([
|
|
927
|
+
prop({
|
|
928
|
+
type: "enum",
|
|
929
|
+
default: "mp4",
|
|
930
|
+
title: "Format",
|
|
931
|
+
description: "Output container. mp4 is H.264. webm is VP9, mov is ProRes, and png_sequence is a zip of PNGs with a manifest.json — those three carry alpha.",
|
|
932
|
+
values: [...TIMELINE_OUTPUT_FORMATS]
|
|
933
|
+
})
|
|
934
|
+
], RenderTimelineNode.prototype, "format", void 0);
|
|
935
|
+
__decorate([
|
|
936
|
+
prop({
|
|
937
|
+
type: "bool",
|
|
938
|
+
default: false,
|
|
939
|
+
title: "Transparent background",
|
|
940
|
+
description: "Composite over a transparent ground and keep the alpha channel. Needs a format that carries alpha: " +
|
|
941
|
+
`${ALPHA_OUTPUT_FORMATS.join(", ")}.`
|
|
942
|
+
})
|
|
943
|
+
], RenderTimelineNode.prototype, "alpha", void 0);
|
|
944
|
+
__decorate([
|
|
945
|
+
prop({
|
|
946
|
+
type: "str",
|
|
947
|
+
default: "",
|
|
948
|
+
title: "Video codec",
|
|
949
|
+
description: "Override the container's default video encoder (mp4 only — webm pins VP9 and mov pins ProRes, the codecs that carry their alpha). Empty uses the default."
|
|
950
|
+
})
|
|
951
|
+
], RenderTimelineNode.prototype, "video_codec", void 0);
|
|
952
|
+
__decorate([
|
|
953
|
+
prop({
|
|
954
|
+
type: "int",
|
|
955
|
+
default: 1,
|
|
956
|
+
title: "Motion blur samples",
|
|
957
|
+
description: "Average this many sub-frame instants into every frame, so fast motion " +
|
|
958
|
+
"smears instead of strobing. 1 is off. Each extra sample costs another " +
|
|
959
|
+
"full render pass.",
|
|
960
|
+
min: 1,
|
|
961
|
+
max: MAX_MOTION_BLUR_SAMPLES
|
|
962
|
+
})
|
|
963
|
+
], RenderTimelineNode.prototype, "motion_blur_samples", void 0);
|
|
964
|
+
__decorate([
|
|
965
|
+
prop({
|
|
966
|
+
type: "float",
|
|
967
|
+
default: DEFAULT_SHUTTER_ANGLE,
|
|
968
|
+
title: "Shutter angle",
|
|
969
|
+
description: "How far the shutter opens, in degrees of one frame. 180 is the film " +
|
|
970
|
+
"convention (open for half the frame); 360 smears across the whole " +
|
|
971
|
+
"frame. Ignored unless motion blur samples is above 1.",
|
|
972
|
+
min: 0,
|
|
973
|
+
max: 360
|
|
974
|
+
})
|
|
975
|
+
], RenderTimelineNode.prototype, "shutter_angle", void 0);
|
|
976
|
+
__decorate([
|
|
977
|
+
prop({
|
|
978
|
+
type: "int",
|
|
979
|
+
default: 0,
|
|
980
|
+
title: "Bitrate",
|
|
981
|
+
description: "Target video bitrate in bits per second. 0 leaves the rate to the encoder.",
|
|
982
|
+
min: 0
|
|
983
|
+
})
|
|
984
|
+
], RenderTimelineNode.prototype, "bitrate", void 0);
|
|
985
|
+
__decorate([
|
|
986
|
+
prop({
|
|
987
|
+
type: "float",
|
|
988
|
+
default: 1,
|
|
989
|
+
title: "Preview scale",
|
|
990
|
+
description: "Render at this fraction of the sequence size. 0.5 is a quarter of the pixels and roughly a quarter of the time — a draft to look at before spending on the full render. 1 is full size."
|
|
991
|
+
})
|
|
992
|
+
], RenderTimelineNode.prototype, "preview_scale", void 0);
|
|
993
|
+
export class TimelineTranscriptNode extends BaseNode {
|
|
994
|
+
static nodeType = "nodetool.timeline.Transcript";
|
|
995
|
+
static title = "Timeline Transcript";
|
|
996
|
+
static description = "Extract the transcript of a timeline sequence as text.\n timeline, transcript, text, script, voiceover\n\n Use cases:\n - Summarize or rewrite an edit's narration with an LLM\n - Generate titles, descriptions, or chapters from the spoken script\n - Translate a voiceover script before re-recording with TTS";
|
|
997
|
+
static metadataOutputTypes = {
|
|
998
|
+
text: "str",
|
|
999
|
+
lines: "list[str]"
|
|
1000
|
+
};
|
|
1001
|
+
static inlineFields = ["timeline"];
|
|
1002
|
+
static inputFields = ["timeline"];
|
|
1003
|
+
async process(context) {
|
|
1004
|
+
const seq = await loadTimelineSequence(this.timeline, context);
|
|
1005
|
+
const lines = (seq.transcript ?? []).map((line) => line.text);
|
|
1006
|
+
return { text: lines.join("\n"), lines };
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
__decorate([
|
|
1010
|
+
prop({
|
|
1011
|
+
type: "timeline",
|
|
1012
|
+
default: { type: "timeline", id: null, data: null },
|
|
1013
|
+
title: "Timeline",
|
|
1014
|
+
description: "The timeline sequence to read the transcript from."
|
|
1015
|
+
})
|
|
1016
|
+
], TimelineTranscriptNode.prototype, "timeline", void 0);
|
|
1017
|
+
/** Content types AddClips falls back to when the bytes say nothing. */
|
|
1018
|
+
const CLIP_CONTENT_TYPE_FALLBACK = {
|
|
1019
|
+
image: "image/png",
|
|
1020
|
+
audio: "audio/wav",
|
|
1021
|
+
video: "video/mp4"
|
|
1022
|
+
};
|
|
1023
|
+
const CLIP_CONTENT_TYPE_BY_EXTENSION = {
|
|
1024
|
+
jpg: "image/jpeg",
|
|
1025
|
+
jpeg: "image/jpeg",
|
|
1026
|
+
png: "image/png",
|
|
1027
|
+
gif: "image/gif",
|
|
1028
|
+
webp: "image/webp",
|
|
1029
|
+
wav: "audio/wav",
|
|
1030
|
+
mp3: "audio/mpeg",
|
|
1031
|
+
ogg: "audio/ogg",
|
|
1032
|
+
flac: "audio/x-flac",
|
|
1033
|
+
m4a: "audio/x-m4a",
|
|
1034
|
+
mp4: "video/mp4",
|
|
1035
|
+
webm: "video/webm",
|
|
1036
|
+
mov: "video/quicktime"
|
|
1037
|
+
};
|
|
1038
|
+
const startsWithBytes = (bytes, ascii, at = 0) => {
|
|
1039
|
+
if (bytes.length < at + ascii.length)
|
|
1040
|
+
return false;
|
|
1041
|
+
for (let i = 0; i < ascii.length; i++) {
|
|
1042
|
+
if (bytes[at + i] !== ascii.charCodeAt(i))
|
|
1043
|
+
return false;
|
|
1044
|
+
}
|
|
1045
|
+
return true;
|
|
1046
|
+
};
|
|
1047
|
+
/** The content type the bytes themselves declare, or null when unrecognized. */
|
|
1048
|
+
function sniffContentType(mediaType, bytes) {
|
|
1049
|
+
if (startsWithBytes(bytes, "RIFF") && startsWithBytes(bytes, "WAVE", 8)) {
|
|
1050
|
+
return "audio/wav";
|
|
1051
|
+
}
|
|
1052
|
+
if (startsWithBytes(bytes, "RIFF") && startsWithBytes(bytes, "WEBP", 8)) {
|
|
1053
|
+
return "image/webp";
|
|
1054
|
+
}
|
|
1055
|
+
if (startsWithBytes(bytes, "ID3"))
|
|
1056
|
+
return "audio/mpeg";
|
|
1057
|
+
// MPEG audio frame sync: 11 set bits. Two bytes is weak evidence, so trust
|
|
1058
|
+
// it only where the clip is already known to be audio.
|
|
1059
|
+
if (mediaType === "audio" &&
|
|
1060
|
+
bytes.length >= 2 &&
|
|
1061
|
+
bytes[0] === 0xff &&
|
|
1062
|
+
(bytes[1] & 0xe0) === 0xe0) {
|
|
1063
|
+
return "audio/mpeg";
|
|
1064
|
+
}
|
|
1065
|
+
if (startsWithBytes(bytes, "OggS"))
|
|
1066
|
+
return "audio/ogg";
|
|
1067
|
+
if (startsWithBytes(bytes, "fLaC"))
|
|
1068
|
+
return "audio/x-flac";
|
|
1069
|
+
if (startsWithBytes(bytes, "\x89PNG"))
|
|
1070
|
+
return "image/png";
|
|
1071
|
+
if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xd8) {
|
|
1072
|
+
return "image/jpeg";
|
|
1073
|
+
}
|
|
1074
|
+
if (startsWithBytes(bytes, "GIF8"))
|
|
1075
|
+
return "image/gif";
|
|
1076
|
+
// Matroska/WebM EBML header.
|
|
1077
|
+
if (bytes.length >= 4 &&
|
|
1078
|
+
bytes[0] === 0x1a &&
|
|
1079
|
+
bytes[1] === 0x45 &&
|
|
1080
|
+
bytes[2] === 0xdf &&
|
|
1081
|
+
bytes[3] === 0xa3) {
|
|
1082
|
+
return "video/webm";
|
|
1083
|
+
}
|
|
1084
|
+
if (startsWithBytes(bytes, "ftyp", 4)) {
|
|
1085
|
+
if (startsWithBytes(bytes, "qt", 8))
|
|
1086
|
+
return "video/quicktime";
|
|
1087
|
+
if (startsWithBytes(bytes, "M4A", 8))
|
|
1088
|
+
return "audio/x-m4a";
|
|
1089
|
+
return "video/mp4";
|
|
1090
|
+
}
|
|
1091
|
+
return null;
|
|
1092
|
+
}
|
|
1093
|
+
/**
|
|
1094
|
+
* The content type to store a clip's bytes under.
|
|
1095
|
+
*
|
|
1096
|
+
* The stored asset's file extension and the `Content-Type` the browser is
|
|
1097
|
+
* served both come from this, so guessing by media kind alone mislabels
|
|
1098
|
+
* anything that is not the kind's default container: a WAV voiceover saved as
|
|
1099
|
+
* `audio/mpeg` reaches the player as a `.mp3` it cannot decode. Read the bytes
|
|
1100
|
+
* first, then the source URI's extension, and only then fall back to the kind.
|
|
1101
|
+
*/
|
|
1102
|
+
export function clipContentType(mediaType, bytes, uri) {
|
|
1103
|
+
const sniffed = sniffContentType(mediaType, bytes);
|
|
1104
|
+
if (sniffed)
|
|
1105
|
+
return sniffed;
|
|
1106
|
+
const extension = (uri ?? "").toLowerCase().split(/[?#]/)[0].split(".").pop();
|
|
1107
|
+
const byExtension = extension
|
|
1108
|
+
? CLIP_CONTENT_TYPE_BY_EXTENSION[extension]
|
|
1109
|
+
: undefined;
|
|
1110
|
+
if (byExtension)
|
|
1111
|
+
return byExtension;
|
|
1112
|
+
return CLIP_CONTENT_TYPE_FALLBACK[mediaType] ?? "application/octet-stream";
|
|
1113
|
+
}
|
|
1114
|
+
export class AddClipsToTimelineNode extends BaseNode {
|
|
1115
|
+
static nodeType = "nodetool.timeline.AddClips";
|
|
1116
|
+
static title = "Add Clips To Timeline";
|
|
1117
|
+
static description = "Append media to a timeline sequence as clips (videos and images on the video track, audio on the audio track). Creates a new timeline when none is given.\n timeline, clips, append, assemble, storyboard\n\n Use cases:\n - Assemble generated shots into an editable rough cut\n - Turn a storyboard's images into an animatic\n - Add a generated voiceover or soundtrack under an edit";
|
|
1118
|
+
static requiredRuntimes = ["ffmpeg"];
|
|
1119
|
+
static metadataOutputTypes = {
|
|
1120
|
+
output: "timeline"
|
|
1121
|
+
};
|
|
1122
|
+
static inlineFields = ["name"];
|
|
1123
|
+
static inputFields = ["timeline", "clips"];
|
|
1124
|
+
async process(context) {
|
|
1125
|
+
if (!context) {
|
|
1126
|
+
throw new Error("AddClipsToTimeline requires a processing context");
|
|
1127
|
+
}
|
|
1128
|
+
const items = (Array.isArray(this.clips) ? this.clips : [])
|
|
1129
|
+
.filter((item) => !!item && typeof item === "object")
|
|
1130
|
+
.filter((item) => ["image", "video", "audio"].includes(String(item.type)));
|
|
1131
|
+
if (items.length === 0) {
|
|
1132
|
+
throw new Error("AddClipsToTimeline: no image/video/audio clips given");
|
|
1133
|
+
}
|
|
1134
|
+
const timelineRef = (this.timeline ?? {});
|
|
1135
|
+
const seq = timelineRef.id
|
|
1136
|
+
? await loadTimelineSequence(timelineRef, context)
|
|
1137
|
+
: makeSequence({
|
|
1138
|
+
name: String(this.name ?? "Untitled video"),
|
|
1139
|
+
projectId: "default"
|
|
1140
|
+
});
|
|
1141
|
+
const ensureTrack = (type) => {
|
|
1142
|
+
const existing = seq.tracks.find((t) => t.type === type);
|
|
1143
|
+
if (existing)
|
|
1144
|
+
return existing;
|
|
1145
|
+
const track = makeTrack({
|
|
1146
|
+
type,
|
|
1147
|
+
name: type === "video" ? "Video" : "Audio",
|
|
1148
|
+
index: seq.tracks.length
|
|
1149
|
+
});
|
|
1150
|
+
seq.tracks.push(track);
|
|
1151
|
+
return track;
|
|
1152
|
+
};
|
|
1153
|
+
const trackEnd = (trackId) => seq.clips
|
|
1154
|
+
.filter((c) => c.trackId === trackId)
|
|
1155
|
+
.reduce((end, c) => Math.max(end, c.startMs + c.durationMs), 0);
|
|
1156
|
+
const workDir = await fs.mkdtemp(path.join(os.tmpdir(), "nodetool-timeline-add-"));
|
|
1157
|
+
try {
|
|
1158
|
+
const cursor = {};
|
|
1159
|
+
for (const [i, item] of items.entries()) {
|
|
1160
|
+
const mediaType = String(item.type);
|
|
1161
|
+
const bytes = await loadMediaRefBytes(item, context);
|
|
1162
|
+
let assetId = isNonEmptyString(item.asset_id)
|
|
1163
|
+
? item.asset_id
|
|
1164
|
+
: null;
|
|
1165
|
+
if (!assetId) {
|
|
1166
|
+
if (!bytes) {
|
|
1167
|
+
console.warn(`AddClipsToTimeline: skipping clip ${i} — media has no resolvable bytes`);
|
|
1168
|
+
continue;
|
|
1169
|
+
}
|
|
1170
|
+
const contentType = clipContentType(mediaType, bytes, isString(item.uri) ? item.uri : undefined);
|
|
1171
|
+
const created = (await context.createAsset({
|
|
1172
|
+
name: `clip-${i + 1}`,
|
|
1173
|
+
contentType,
|
|
1174
|
+
content: bytes
|
|
1175
|
+
}));
|
|
1176
|
+
assetId = created.id;
|
|
1177
|
+
}
|
|
1178
|
+
let durationMs = Number(this.image_duration_ms ?? 3000);
|
|
1179
|
+
if (mediaType !== "image") {
|
|
1180
|
+
if (!bytes) {
|
|
1181
|
+
console.warn(`AddClipsToTimeline: skipping clip ${i} — cannot probe duration without media bytes`);
|
|
1182
|
+
continue;
|
|
1183
|
+
}
|
|
1184
|
+
const probePath = path.join(workDir, `probe_${i}`);
|
|
1185
|
+
await fs.writeFile(probePath, bytes);
|
|
1186
|
+
const seconds = await ffprobeDuration(probePath);
|
|
1187
|
+
if (seconds <= 0) {
|
|
1188
|
+
console.warn(`AddClipsToTimeline: skipping clip ${i} — could not determine duration`);
|
|
1189
|
+
continue;
|
|
1190
|
+
}
|
|
1191
|
+
durationMs = Math.round(seconds * 1000);
|
|
1192
|
+
}
|
|
1193
|
+
const track = ensureTrack(mediaType === "audio" ? "audio" : "video");
|
|
1194
|
+
const startMs = cursor[track.id] ?? trackEnd(track.id);
|
|
1195
|
+
seq.clips.push(makeClip({
|
|
1196
|
+
trackId: track.id,
|
|
1197
|
+
name: `Clip ${seq.clips.length + 1}`,
|
|
1198
|
+
startMs,
|
|
1199
|
+
durationMs,
|
|
1200
|
+
mediaType,
|
|
1201
|
+
sourceType: "imported",
|
|
1202
|
+
status: "generated",
|
|
1203
|
+
currentAssetId: assetId
|
|
1204
|
+
}));
|
|
1205
|
+
cursor[track.id] = startMs + durationMs;
|
|
1206
|
+
}
|
|
1207
|
+
}
|
|
1208
|
+
finally {
|
|
1209
|
+
await fs.rm(workDir, { recursive: true, force: true });
|
|
1210
|
+
}
|
|
1211
|
+
seq.durationMs = seq.clips.reduce((end, c) => Math.max(end, c.startMs + c.durationMs), 0);
|
|
1212
|
+
const saved = (timelineRef.id
|
|
1213
|
+
? await context.updateTimelineSequence(seq.id, seq)
|
|
1214
|
+
: await context.createTimelineSequence(seq));
|
|
1215
|
+
if (!saved) {
|
|
1216
|
+
throw new Error("AddClipsToTimeline: failed to save timeline sequence");
|
|
1217
|
+
}
|
|
1218
|
+
return { output: { type: "timeline", id: saved.id } };
|
|
1219
|
+
}
|
|
1220
|
+
}
|
|
1221
|
+
__decorate([
|
|
1222
|
+
prop({
|
|
1223
|
+
type: "timeline",
|
|
1224
|
+
default: { type: "timeline", id: null, data: null },
|
|
1225
|
+
title: "Timeline",
|
|
1226
|
+
description: "Timeline to append to. Leave empty to create a new timeline."
|
|
1227
|
+
})
|
|
1228
|
+
], AddClipsToTimelineNode.prototype, "timeline", void 0);
|
|
1229
|
+
__decorate([
|
|
1230
|
+
prop({
|
|
1231
|
+
type: "list",
|
|
1232
|
+
default: [],
|
|
1233
|
+
title: "Clips",
|
|
1234
|
+
description: "Media to append: image, video, and audio references."
|
|
1235
|
+
})
|
|
1236
|
+
], AddClipsToTimelineNode.prototype, "clips", void 0);
|
|
1237
|
+
__decorate([
|
|
1238
|
+
prop({
|
|
1239
|
+
type: "str",
|
|
1240
|
+
default: "Untitled video",
|
|
1241
|
+
title: "Name",
|
|
1242
|
+
description: "Name for the timeline when a new one is created."
|
|
1243
|
+
})
|
|
1244
|
+
], AddClipsToTimelineNode.prototype, "name", void 0);
|
|
1245
|
+
__decorate([
|
|
1246
|
+
prop({
|
|
1247
|
+
type: "int",
|
|
1248
|
+
default: 3000,
|
|
1249
|
+
title: "Image duration (ms)",
|
|
1250
|
+
description: "Clip duration for still images, in milliseconds.",
|
|
1251
|
+
min: 100,
|
|
1252
|
+
max: 600000
|
|
1253
|
+
})
|
|
1254
|
+
], AddClipsToTimelineNode.prototype, "image_duration_ms", void 0);
|
|
1255
|
+
// ── Template derivations (design docs/graph-resources/design.md § 4.4) ──
|
|
1256
|
+
const timelineRefDefault = {
|
|
1257
|
+
type: "timeline",
|
|
1258
|
+
id: null,
|
|
1259
|
+
data: null
|
|
1260
|
+
};
|
|
1261
|
+
export class FillTimelineTextNode extends BaseNode {
|
|
1262
|
+
static nodeType = "nodetool.timeline.FillTimelineText";
|
|
1263
|
+
static title = "Fill Timeline Text";
|
|
1264
|
+
static description = "Fill a template cut's {{key}} placeholders — in text clips and in captions — from a value bag, and save the result as a new timeline. The template is never written. A key with no value stays in the frame and is reported.\n timeline, template, text, fill, batch\n\n Use cases:\n - Stamp a per-SKU name and price onto an approved cut\n - Localize an overlay without re-editing the timeline\n - Produce one variant per row of a feed";
|
|
1265
|
+
static metadataOutputTypes = {
|
|
1266
|
+
timeline: "timeline",
|
|
1267
|
+
filled: "list[str]",
|
|
1268
|
+
unresolved: "list[str]"
|
|
1269
|
+
};
|
|
1270
|
+
static inlineFields = ["timeline"];
|
|
1271
|
+
static inputFields = ["timeline", "values"];
|
|
1272
|
+
async process(context) {
|
|
1273
|
+
if (!context) {
|
|
1274
|
+
throw new Error("FillTimelineText requires a processing context");
|
|
1275
|
+
}
|
|
1276
|
+
const source = await loadTimelineSequence(this.timeline, context);
|
|
1277
|
+
const result = fillTimelineText(source, stringValues(this.values));
|
|
1278
|
+
// `fillTimelineText` is a text edit, so it leaves identity alone — an
|
|
1279
|
+
// editor may run it on the sequence it has open. This node derives a row
|
|
1280
|
+
// instead, so it mints the id and the lineage here.
|
|
1281
|
+
const now = new Date().toISOString();
|
|
1282
|
+
const saved = (await context.createTimelineSequence({
|
|
1283
|
+
...result.sequence,
|
|
1284
|
+
id: createTimeOrderedUuid(),
|
|
1285
|
+
templateId: source.id,
|
|
1286
|
+
name: this.name?.trim() || source.name,
|
|
1287
|
+
createdAt: now,
|
|
1288
|
+
updatedAt: now
|
|
1289
|
+
}));
|
|
1290
|
+
if (!saved) {
|
|
1291
|
+
throw new Error("FillTimelineText: failed to create the timeline");
|
|
1292
|
+
}
|
|
1293
|
+
return {
|
|
1294
|
+
timeline: { type: "timeline", id: saved.id },
|
|
1295
|
+
filled: result.filled,
|
|
1296
|
+
unresolved: result.unresolved
|
|
1297
|
+
};
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
__decorate([
|
|
1301
|
+
prop({
|
|
1302
|
+
type: "timeline",
|
|
1303
|
+
default: timelineRefDefault,
|
|
1304
|
+
title: "Timeline",
|
|
1305
|
+
description: "The template cut to fill."
|
|
1306
|
+
})
|
|
1307
|
+
], FillTimelineTextNode.prototype, "timeline", void 0);
|
|
1308
|
+
__decorate([
|
|
1309
|
+
prop({
|
|
1310
|
+
type: "dict",
|
|
1311
|
+
default: {},
|
|
1312
|
+
title: "Values",
|
|
1313
|
+
description: "Value per placeholder key, e.g. {name: \"Aero 9\"}."
|
|
1314
|
+
})
|
|
1315
|
+
], FillTimelineTextNode.prototype, "values", void 0);
|
|
1316
|
+
__decorate([
|
|
1317
|
+
prop({
|
|
1318
|
+
type: "str",
|
|
1319
|
+
default: "",
|
|
1320
|
+
title: "Name",
|
|
1321
|
+
description: "Name for the new timeline. Defaults to the template's name."
|
|
1322
|
+
})
|
|
1323
|
+
], FillTimelineTextNode.prototype, "name", void 0);
|
|
1324
|
+
export class RetargetTimelineNode extends BaseNode {
|
|
1325
|
+
static nodeType = "nodetool.timeline.RetargetTimeline";
|
|
1326
|
+
static title = "Retarget Timeline";
|
|
1327
|
+
static description = "Derive a new timeline from an approved cut on another aspect ratio, keeping every trim, placement and edit. Cover scales each shot to fill the new frame and reports the clips it crops; contain letterboxes instead. Text sizes follow the short edge. The source cut is never written.\n timeline, aspect ratio, resize, reframe, variant\n\n Use cases:\n - Turn a 16:9 film into 9:16 and 1:1 cuts\n - Produce every placement's ratio from one approved edit\n - See which shots lose their sides before rendering";
|
|
1328
|
+
static metadataOutputTypes = {
|
|
1329
|
+
timeline: "timeline",
|
|
1330
|
+
cropped: "list[str]"
|
|
1331
|
+
};
|
|
1332
|
+
static inlineFields = ["aspect_ratio", "fit", "name"];
|
|
1333
|
+
static inputFields = ["timeline"];
|
|
1334
|
+
async process(context) {
|
|
1335
|
+
if (!context) {
|
|
1336
|
+
throw new Error("RetargetTimeline requires a processing context");
|
|
1337
|
+
}
|
|
1338
|
+
const source = await loadTimelineSequence(this.timeline, context);
|
|
1339
|
+
const ratio = this.aspect_ratio?.trim() || "9:16";
|
|
1340
|
+
const { sequence, croppedClipIds } = retargetSequence(source, ratio, this.fit === "contain" ? "contain" : "cover");
|
|
1341
|
+
const saved = (await context.createTimelineSequence({
|
|
1342
|
+
...sequence,
|
|
1343
|
+
name: this.name?.trim() || `${source.name} ${ratio}`
|
|
1344
|
+
}));
|
|
1345
|
+
if (!saved) {
|
|
1346
|
+
throw new Error("RetargetTimeline: failed to create the timeline");
|
|
1347
|
+
}
|
|
1348
|
+
return {
|
|
1349
|
+
timeline: { type: "timeline", id: saved.id },
|
|
1350
|
+
cropped: croppedClipIds
|
|
1351
|
+
};
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1354
|
+
__decorate([
|
|
1355
|
+
prop({
|
|
1356
|
+
type: "timeline",
|
|
1357
|
+
default: timelineRefDefault,
|
|
1358
|
+
title: "Timeline",
|
|
1359
|
+
description: "The approved cut to retarget."
|
|
1360
|
+
})
|
|
1361
|
+
], RetargetTimelineNode.prototype, "timeline", void 0);
|
|
1362
|
+
__decorate([
|
|
1363
|
+
prop({
|
|
1364
|
+
type: "str",
|
|
1365
|
+
default: "9:16",
|
|
1366
|
+
title: "Aspect Ratio",
|
|
1367
|
+
description: "Target ratio, e.g. 9:16, 1:1, 4:5. The short edge is 1080px."
|
|
1368
|
+
})
|
|
1369
|
+
], RetargetTimelineNode.prototype, "aspect_ratio", void 0);
|
|
1370
|
+
__decorate([
|
|
1371
|
+
prop({
|
|
1372
|
+
type: "enum",
|
|
1373
|
+
default: "cover",
|
|
1374
|
+
title: "Fit",
|
|
1375
|
+
description: "cover fills the new frame and crops; contain fits it and letterboxes.",
|
|
1376
|
+
values: ["cover", "contain"]
|
|
1377
|
+
})
|
|
1378
|
+
], RetargetTimelineNode.prototype, "fit", void 0);
|
|
1379
|
+
__decorate([
|
|
1380
|
+
prop({
|
|
1381
|
+
type: "str",
|
|
1382
|
+
default: "",
|
|
1383
|
+
title: "Name",
|
|
1384
|
+
description: "Name for the new timeline. Defaults to the source's name plus the ratio."
|
|
1385
|
+
})
|
|
1386
|
+
], RetargetTimelineNode.prototype, "name", void 0);
|
|
1387
|
+
export const TIMELINE_NODES = tagAsNode([
|
|
1388
|
+
RenderTimelineNode,
|
|
1389
|
+
TimelineTranscriptNode,
|
|
1390
|
+
AddClipsToTimelineNode,
|
|
1391
|
+
// Tagged first, so `tagAsNode` leaves them alone: neither derivation shells
|
|
1392
|
+
// out to ffmpeg or writes a temp file, and both need the model interfaces.
|
|
1393
|
+
...tagAsServer([FillTimelineTextNode, RetargetTimelineNode])
|
|
1394
|
+
]);
|
|
1395
|
+
//# sourceMappingURL=timeline.js.map
|