@nodetool-ai/video-nodes 0.8.2 → 0.8.4-a13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.nodetool-build-stamp +1 -1
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +19 -0
- package/dist/index.js.map +1 -1
- package/dist/nodes/ffmpeg-helpers.d.ts +2 -1
- package/dist/nodes/ffmpeg-helpers.d.ts.map +1 -1
- package/dist/nodes/ffmpeg-helpers.js +4 -5
- package/dist/nodes/ffmpeg-helpers.js.map +1 -1
- package/dist/nodes/model3d/convert.d.ts +2 -1
- package/dist/nodes/model3d/convert.d.ts.map +1 -1
- package/dist/nodes/model3d/convert.js +3 -3
- package/dist/nodes/model3d/convert.js.map +1 -1
- package/dist/nodes/model3d/render3d-headless.d.ts +15 -5
- package/dist/nodes/model3d/render3d-headless.d.ts.map +1 -1
- package/dist/nodes/model3d/render3d-headless.js +154 -36
- package/dist/nodes/model3d/render3d-headless.js.map +1 -1
- package/dist/nodes/model3d/types.d.ts +1 -0
- package/dist/nodes/model3d/types.d.ts.map +1 -1
- package/dist/nodes/model3d/utils.d.ts +6 -13
- package/dist/nodes/model3d/utils.d.ts.map +1 -1
- package/dist/nodes/model3d/utils.js +22 -65
- package/dist/nodes/model3d/utils.js.map +1 -1
- package/dist/nodes/storyboard.d.ts +36 -7
- package/dist/nodes/storyboard.d.ts.map +1 -1
- package/dist/nodes/storyboard.js +157 -7
- package/dist/nodes/storyboard.js.map +1 -1
- package/dist/nodes/timeline/browserRender.d.ts +83 -0
- package/dist/nodes/timeline/browserRender.d.ts.map +1 -0
- package/dist/nodes/timeline/browserRender.js +541 -0
- package/dist/nodes/timeline/browserRender.js.map +1 -0
- package/dist/nodes/timeline/browserSources.d.ts +26 -0
- package/dist/nodes/timeline/browserSources.d.ts.map +1 -0
- package/dist/nodes/timeline/browserSources.js +83 -0
- package/dist/nodes/timeline/browserSources.js.map +1 -0
- package/dist/nodes/timeline/compositeRender.d.ts.map +1 -1
- package/dist/nodes/timeline/compositeRender.js +31 -15
- package/dist/nodes/timeline/compositeRender.js.map +1 -1
- package/dist/nodes/timeline/pngSequence.d.ts +19 -0
- package/dist/nodes/timeline/pngSequence.d.ts.map +1 -1
- package/dist/nodes/timeline/pngSequence.js +21 -6
- package/dist/nodes/timeline/pngSequence.js.map +1 -1
- package/dist/nodes/timeline/rawFrames.d.ts +34 -1
- package/dist/nodes/timeline/rawFrames.d.ts.map +1 -1
- package/dist/nodes/timeline/rawFrames.js +68 -15
- package/dist/nodes/timeline/rawFrames.js.map +1 -1
- package/dist/nodes/timeline/renderPage.d.ts +14 -0
- package/dist/nodes/timeline/renderPage.d.ts.map +1 -0
- package/dist/nodes/timeline/renderPage.js +75 -0
- package/dist/nodes/timeline/renderPage.js.map +1 -0
- package/dist/nodes/timeline.d.ts +21 -24
- package/dist/nodes/timeline.d.ts.map +1 -1
- package/dist/nodes/timeline.js +366 -112
- package/dist/nodes/timeline.js.map +1 -1
- package/dist/nodes/video.d.ts +1 -4
- package/dist/nodes/video.d.ts.map +1 -1
- package/dist/nodes/video.js +21 -3
- package/dist/nodes/video.js.map +1 -1
- package/dist/timeline-render-page.js +6903 -0
- package/package.json +12 -11
package/dist/nodes/timeline.js
CHANGED
|
@@ -2,21 +2,22 @@
|
|
|
2
2
|
* Timeline nodes — operate on persisted timeline sequences (the video
|
|
3
3
|
* editor's documents) referenced by the `timeline` type.
|
|
4
4
|
*
|
|
5
|
-
* RenderTimeline
|
|
6
|
-
* compositor
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
5
|
+
* RenderTimeline runs the editor's own exporter in headless Chromium
|
|
6
|
+
* (`timeline/browserRender.ts`): the compositor, scene model, WebCodecs decode
|
|
7
|
+
* and mux the Export button uses, so a workflow render is the picture the user
|
|
8
|
+
* previewed — clip placement, transforms, opacity, blend modes, transitions,
|
|
9
|
+
* animations, effects, captions, text and shapes, across every visual track —
|
|
10
|
+
* and variable-frame-rate clips decode at their own timestamps.
|
|
11
11
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
12
|
+
* Where no Chromium can be launched, the Node compositor renders instead
|
|
13
|
+
* (`timeline/compositeRender.ts`): the same scene model and compositor through
|
|
14
|
+
* Dawn, with ffmpeg decoding clip media into RGBA and encoding the result. It
|
|
15
|
+
* needs a WebGPU device; a host with no adapter falls back further, to the
|
|
16
|
+
* ffmpeg rough cut — video/image clips normalized and concatenated in start
|
|
17
|
+
* order — which ignores everything above. Each fallback is a `warning` on the
|
|
18
|
+
* job, and `metadata.render_mode` on the output says which path ran:
|
|
19
|
+
* "browser", "composited" or "rough_cut". `NODETOOL_TIMELINE_RENDERER=node`
|
|
20
|
+
* skips the browser.
|
|
20
21
|
*
|
|
21
22
|
* `format` picks the container and `alpha` decides whether the frames keep a
|
|
22
23
|
* transparency channel: `webm` (VP9, `yuva420p`), `mov` (ProRes 4444,
|
|
@@ -32,7 +33,7 @@ var __decorate = (this && this.__decorate) || function (decorators, target, key,
|
|
|
32
33
|
};
|
|
33
34
|
import { BaseNode, prop } from "@nodetool-ai/node-sdk";
|
|
34
35
|
import { loadMediaRefBytes } from "@nodetool-ai/runtime";
|
|
35
|
-
import { createTimeOrderedUuid, DEFAULT_MIDI_INSTRUMENT, encodeWavPcm16, ffmpegFadeCurve, fillTimelineText, resolveClipFades, retargetSequence, hasTimeRemap, sourceRate, makeClip, makeSequence, makeTrack, renderMidiClip, loadSamplerSamples, resolveTempo, timeRemapAudioSegments } from "@nodetool-ai/timeline";
|
|
36
|
+
import { computeModel3DBakeHash, createTimeOrderedUuid, DEFAULT_MIDI_INSTRUMENT, effectiveAssetId, extractedAudioLinkIds as sharedExtractedAudioLinkIds, soundTrackSoloed, videoClipsWithOwnAudio, encodeWavPcm16, ffmpegFadeCurve, fillTimelineText, resolveClipFades, retargetSequence, hasTimeRemap, sourceRate, makeClip, makeSequence, makeTrack, renderMidiClip, loadSamplerSamples, resolveTempo, timeRemapAudioSegments, clipAudioTailSec } from "@nodetool-ai/timeline";
|
|
36
37
|
import { tagAsNode, tagAsServer } from "@nodetool-ai/nodes-utils";
|
|
37
38
|
import { promises as fs } from "node:fs";
|
|
38
39
|
import os from "node:os";
|
|
@@ -40,6 +41,7 @@ import path from "node:path";
|
|
|
40
41
|
import { AssetFiles, assetFileName } from "./timeline/assetFiles.js";
|
|
41
42
|
import { atempoChain, execFfmpeg, execFfprobe, ffprobeDuration, FFMPEG_MAX_BUFFER, MissingBinaryError } from "./ffmpeg-helpers.js";
|
|
42
43
|
import { CompositorUnavailableError, renderTimelineComposited } from "./timeline/compositeRender.js";
|
|
44
|
+
import { BrowserRenderUnavailableError, renderTimelineInBrowser } from "./timeline/browserRender.js";
|
|
43
45
|
import { ALPHA_OUTPUT_FORMATS, MissingEncoderError, resolveTimelineOutput, TIMELINE_OUTPUT_FORMATS } from "./timeline/outputFormats.js";
|
|
44
46
|
import { ffmpegHasEncoder } from "./timeline/rawFrames.js";
|
|
45
47
|
import { stringValues } from "./placeholder-values.js";
|
|
@@ -101,16 +103,20 @@ function renderableVideoClips(seq) {
|
|
|
101
103
|
/**
|
|
102
104
|
* Clips the mix draws from: audio-track clips backed by an asset, and midi
|
|
103
105
|
* clips, whose sound is rendered from the notes they carry rather than fetched
|
|
104
|
-
* (they have no `currentAssetId` at all).
|
|
106
|
+
* (they have no `currentAssetId` at all). An audio clip sounds under the same
|
|
107
|
+
* status rule its picture would be drawn under (`effectiveAssetId`).
|
|
105
108
|
*/
|
|
106
109
|
function mixableAudioClips(seq) {
|
|
107
110
|
const tracks = trackById(seq.tracks);
|
|
111
|
+
const hasSolo = soundTrackSoloed(seq.tracks);
|
|
108
112
|
return seq.clips
|
|
109
113
|
.filter((clip) => {
|
|
110
114
|
const track = tracks.get(clip.trackId);
|
|
111
115
|
if (!track || track.muted === true || clip.muted || clip.hidden) {
|
|
112
116
|
return false;
|
|
113
117
|
}
|
|
118
|
+
if (hasSolo && !track.solo)
|
|
119
|
+
return false;
|
|
114
120
|
if (clip.durationMs <= 0)
|
|
115
121
|
return false;
|
|
116
122
|
if (track.type === "midi") {
|
|
@@ -118,32 +124,13 @@ function mixableAudioClips(seq) {
|
|
|
118
124
|
}
|
|
119
125
|
return (track.type === "audio" &&
|
|
120
126
|
clip.mediaType === "audio" &&
|
|
121
|
-
!!clip
|
|
127
|
+
!!effectiveAssetId(clip));
|
|
122
128
|
})
|
|
123
129
|
.sort((a, b) => a.startMs - b.startMs);
|
|
124
130
|
}
|
|
125
|
-
/**
|
|
126
|
-
* linkIds of audio-track clips that are the detached audio representation of a
|
|
127
|
-
* video clip. The mere presence of such a clip means the linked video's own
|
|
128
|
-
* muxed audio must be suppressed (the editor mutes the video element; audio
|
|
129
|
-
* comes only from audio-track clips). This is intentionally NOT gated on
|
|
130
|
-
* mute/hidden — those govern whether the audio clip is mixed in, not whether
|
|
131
|
-
* the video's embedded audio resurfaces. If the user deletes the audio clip,
|
|
132
|
-
* the store auto-unlinks the lone survivor, clearing the video's linkId, so the
|
|
133
|
-
* video's audio returns on its own.
|
|
134
|
-
*/
|
|
131
|
+
/** The detached-audio linkIds of a sequence; see the shared rule. */
|
|
135
132
|
export function extractedAudioLinkIds(seq) {
|
|
136
|
-
|
|
137
|
-
const ids = new Set();
|
|
138
|
-
for (const clip of seq.clips) {
|
|
139
|
-
const track = tracks.get(clip.trackId);
|
|
140
|
-
if (track?.type === "audio" &&
|
|
141
|
-
clip.mediaType === "audio" &&
|
|
142
|
-
isNonEmptyString(clip.linkId)) {
|
|
143
|
-
ids.add(clip.linkId);
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
return ids;
|
|
133
|
+
return sharedExtractedAudioLinkIds(seq.clips, seq.tracks);
|
|
147
134
|
}
|
|
148
135
|
/** Scale + letterbox to the sequence frame, normalize fps and pixel format. */
|
|
149
136
|
function segmentVideoFilter(width, height, fps) {
|
|
@@ -234,25 +221,11 @@ async function encodeSegment(opts) {
|
|
|
234
221
|
/**
|
|
235
222
|
* Clips whose embedded audio the composited path must mix in itself: the
|
|
236
223
|
* rough cut carried a video clip's audio through its segment, but a
|
|
237
|
-
* frame-by-frame composite has no audio at all.
|
|
238
|
-
*
|
|
239
|
-
* {@link extractedAudioLinkIds}) are left out.
|
|
224
|
+
* frame-by-frame composite has no audio at all. The preview and browser
|
|
225
|
+
* export pick the same clips through `videoClipsWithOwnAudio`.
|
|
240
226
|
*/
|
|
241
227
|
function embeddedAudioClips(seq) {
|
|
242
|
-
|
|
243
|
-
const suppressed = extractedAudioLinkIds(seq);
|
|
244
|
-
return seq.clips
|
|
245
|
-
.filter((clip) => {
|
|
246
|
-
const track = tracks.get(clip.trackId);
|
|
247
|
-
return ((track?.type === "video" || track?.type === "overlay") &&
|
|
248
|
-
track.muted !== true &&
|
|
249
|
-
!clip.muted &&
|
|
250
|
-
clip.mediaType === "video" &&
|
|
251
|
-
!!clip.currentAssetId &&
|
|
252
|
-
clip.durationMs > 0 &&
|
|
253
|
-
!(isString(clip.linkId) && suppressed.has(clip.linkId)));
|
|
254
|
-
})
|
|
255
|
-
.sort((a, b) => a.startMs - b.startMs);
|
|
228
|
+
return videoClipsWithOwnAudio(seq.clips, seq.tracks).sort((a, b) => a.startMs - b.startMs);
|
|
256
229
|
}
|
|
257
230
|
/** Seconds, at millisecond resolution, for an ffmpeg filter argument. */
|
|
258
231
|
function ffSeconds(ms) {
|
|
@@ -305,6 +278,7 @@ function audioClipFilter(clip, inputIndex, label) {
|
|
|
305
278
|
steps.push(`volume=${clip.volumeDb}dB`);
|
|
306
279
|
}
|
|
307
280
|
steps.push(...clipFadeSteps(clip, 0));
|
|
281
|
+
steps.push(...clipEffectChainSteps(clip));
|
|
308
282
|
const delay = Math.max(0, Math.round(clip.startMs));
|
|
309
283
|
steps.push(`adelay=${delay}|${delay}`);
|
|
310
284
|
return `[${inputIndex}:a]${steps.join(",")}[${label}]`;
|
|
@@ -332,7 +306,174 @@ function audioRemapSegmentFilter(clip, segment, inputIndex, label) {
|
|
|
332
306
|
steps.push(`adelay=${delay}|${delay}`);
|
|
333
307
|
return `[${inputIndex}:a]${steps.join(",")}[${label}]`;
|
|
334
308
|
}
|
|
335
|
-
/**
|
|
309
|
+
/** Clamp for an ffmpeg option that refuses values outside its range. */
|
|
310
|
+
function clampTo(value, min, max) {
|
|
311
|
+
return Math.min(max, Math.max(min, value));
|
|
312
|
+
}
|
|
313
|
+
/** Sample rate a track's effect chain runs at, the mix's own `-ar`. */
|
|
314
|
+
const EFFECT_SAMPLE_RATE = 48000;
|
|
315
|
+
/** A biquad frequency ffmpeg accepts, held below Nyquist as Web Audio holds it. */
|
|
316
|
+
function biquadHz(frequency) {
|
|
317
|
+
return clampTo(frequency, 1, EFFECT_SAMPLE_RATE / 2 - 1);
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* Web Audio's compressor adds makeup gain of its own: the gain that brings a
|
|
321
|
+
* full-scale input back to full scale, raised to the power 0.6. ffmpeg's
|
|
322
|
+
* `acompressor` adds none, so it is computed here from the curve at 0 dBFS
|
|
323
|
+
* with a quadratic soft knee, which is close to but not exactly the spec's.
|
|
324
|
+
*/
|
|
325
|
+
function compressorMakeupGain(e) {
|
|
326
|
+
const threshold = clampTo(e.thresholdDb, -100, 0);
|
|
327
|
+
const ratio = clampTo(e.ratio, 1, 20);
|
|
328
|
+
const knee = clampTo(e.kneeDb, 0, 40);
|
|
329
|
+
const over = -threshold;
|
|
330
|
+
let gainDb;
|
|
331
|
+
if (2 * over < -knee) {
|
|
332
|
+
gainDb = 0;
|
|
333
|
+
}
|
|
334
|
+
else if (2 * Math.abs(over) <= knee) {
|
|
335
|
+
gainDb = ((1 / ratio - 1) * (over + knee / 2) ** 2) / (2 * Math.max(knee, 1e-6));
|
|
336
|
+
}
|
|
337
|
+
else {
|
|
338
|
+
gainDb = threshold + over / ratio;
|
|
339
|
+
}
|
|
340
|
+
return clampTo(Math.pow(10, (-gainDb * 0.6) / 20), 1, 64);
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* ffmpeg steps for one enabled effect, matching the Web Audio node
|
|
344
|
+
* `AudioGraph.buildEffectUnit` builds for the preview. The shelves use Web
|
|
345
|
+
* Audio's fixed slope of 1, a lowpass/highpass Q is in dB as it is there, and
|
|
346
|
+
* a bandpass Q is linear. The compressor detects peaks; its knee is capped at
|
|
347
|
+
* the 18 dB `acompressor` allows.
|
|
348
|
+
*/
|
|
349
|
+
function audioEffectSteps(effect) {
|
|
350
|
+
switch (effect.type) {
|
|
351
|
+
case "gain":
|
|
352
|
+
return [`volume=${effect.gainDb}dB`];
|
|
353
|
+
case "eq3":
|
|
354
|
+
return [
|
|
355
|
+
`lowshelf=f=${biquadHz(effect.lowFreq)}:g=${effect.lowGainDb}:t=s:w=1`,
|
|
356
|
+
`equalizer=f=${biquadHz(effect.midFreq)}:t=q:w=${Math.max(0.0001, effect.midQ)}:g=${effect.midGainDb}`,
|
|
357
|
+
`highshelf=f=${biquadHz(effect.highFreq)}:g=${effect.highGainDb}:t=s:w=1`
|
|
358
|
+
];
|
|
359
|
+
case "filter": {
|
|
360
|
+
const q = effect.mode === "bandpass"
|
|
361
|
+
? Math.max(0.0001, effect.q)
|
|
362
|
+
: Math.max(0.0001, Math.pow(10, effect.q / 20));
|
|
363
|
+
return [`${effect.mode}=f=${biquadHz(effect.frequency)}:t=q:w=${q}`];
|
|
364
|
+
}
|
|
365
|
+
case "compressor":
|
|
366
|
+
return [
|
|
367
|
+
"acompressor=" +
|
|
368
|
+
[
|
|
369
|
+
`threshold=${clampTo(Math.pow(10, effect.thresholdDb / 20), 0.000976563, 1)}`,
|
|
370
|
+
`ratio=${clampTo(effect.ratio, 1, 20)}`,
|
|
371
|
+
`attack=${clampTo(effect.attackMs, 0.01, 2000)}`,
|
|
372
|
+
`release=${clampTo(effect.releaseMs, 0.01, 9000)}`,
|
|
373
|
+
`knee=${clampTo(Math.pow(10, effect.kneeDb / 20), 1, 8)}`,
|
|
374
|
+
`makeup=${compressorMakeupGain(effect)}`,
|
|
375
|
+
"detection=peak"
|
|
376
|
+
].join(":")
|
|
377
|
+
];
|
|
378
|
+
default:
|
|
379
|
+
return [];
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
/** Most taps an `aecho` is given, so a long feedback echo stays cheap. */
|
|
383
|
+
const MAX_ECHO_TAPS = 32;
|
|
384
|
+
/** A tap quieter than this (-60 dB) is left out. */
|
|
385
|
+
const SILENT_TAP_GAIN = 0.001;
|
|
386
|
+
/** Taps the reverb approximation spreads over its decay. */
|
|
387
|
+
const REVERB_TAPS = 24;
|
|
388
|
+
/** An `aecho` from a dry gain and taps of `[delayMs, gain]`. */
|
|
389
|
+
function aechoStep(dryGain, taps) {
|
|
390
|
+
const delays = taps.map(([ms]) => clampTo(Math.round(ms), 1, 90000));
|
|
391
|
+
const decays = taps.map(([, gain]) => clampTo(gain, 0.0001, 1).toFixed(5));
|
|
392
|
+
return `aecho=${clampTo(dryGain, 0.001, 1).toFixed(5)}:1:${delays.join("|")}:${decays.join("|")}`;
|
|
393
|
+
}
|
|
394
|
+
/**
|
|
395
|
+
* ffmpeg steps for one clip effect, matching `audioEffectNodes.ts` in the
|
|
396
|
+
* browser. The track DSP types reuse {@link audioEffectSteps}.
|
|
397
|
+
*
|
|
398
|
+
* An echo is a delay line feeding itself; unrolled, it is a tap at every
|
|
399
|
+
* multiple of the delay, each `feedback` times the one before, which is what
|
|
400
|
+
* `aecho` plays. A reverb in the browser convolves with decaying noise, and
|
|
401
|
+
* ffmpeg has no convolver without a second input stream, so here it is a
|
|
402
|
+
* spread of taps on the same -60 dB envelope: the same decay and level, with
|
|
403
|
+
* a grainier tail.
|
|
404
|
+
*/
|
|
405
|
+
function clipAudioEffectSteps(effect) {
|
|
406
|
+
switch (effect.type) {
|
|
407
|
+
case "echo": {
|
|
408
|
+
const mix = clampTo(effect.mix, 0, 1);
|
|
409
|
+
const feedback = clampTo(effect.feedback, 0, 0.95);
|
|
410
|
+
const taps = [];
|
|
411
|
+
for (let k = 1; k <= MAX_ECHO_TAPS; k += 1) {
|
|
412
|
+
const gain = mix * Math.pow(feedback, k - 1);
|
|
413
|
+
if (k > 1 && gain < SILENT_TAP_GAIN)
|
|
414
|
+
break;
|
|
415
|
+
taps.push([effect.delayMs * k, gain]);
|
|
416
|
+
}
|
|
417
|
+
return [aechoStep(1 - mix, taps)];
|
|
418
|
+
}
|
|
419
|
+
case "reverb": {
|
|
420
|
+
const mix = clampTo(effect.mix, 0, 1);
|
|
421
|
+
const decayMs = Math.max(50, effect.decaySec * 1000);
|
|
422
|
+
const times = Array.from({ length: REVERB_TAPS }, (_, i) => 10 + decayMs * 0.8 * Math.pow((i + 1) / REVERB_TAPS, 1.6));
|
|
423
|
+
// ln(1000): -60 dB at the end of the decay.
|
|
424
|
+
const raw = times.map((ms) => Math.exp((-Math.log(1000) * ms) / decayMs));
|
|
425
|
+
const norm = Math.sqrt(raw.reduce((sum, g) => sum + g * g, 0));
|
|
426
|
+
return [
|
|
427
|
+
aechoStep(1 - mix, times.map((ms, i) => [ms, (mix * raw[i]) / norm]))
|
|
428
|
+
];
|
|
429
|
+
}
|
|
430
|
+
case "distortion": {
|
|
431
|
+
// tanh(k·x) / tanh(k), as the browser's wave shaper curve.
|
|
432
|
+
const drive = Math.pow(10, Math.max(0, effect.driveDb) / 20);
|
|
433
|
+
const normDb = -20 * Math.log10(Math.tanh(drive));
|
|
434
|
+
return [
|
|
435
|
+
`volume=${Math.max(0, effect.driveDb)}dB`,
|
|
436
|
+
"asoftclip=type=tanh",
|
|
437
|
+
`volume=${(effect.outputDb + normDb).toFixed(3)}dB`
|
|
438
|
+
];
|
|
439
|
+
}
|
|
440
|
+
default:
|
|
441
|
+
return audioEffectSteps(effect);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* A clip's real-time effects, after its gain and fades. The stream is padded
|
|
446
|
+
* by the effects' tail first, so an echo or reverb rings on past the clip's
|
|
447
|
+
* last sample instead of stopping with it.
|
|
448
|
+
*/
|
|
449
|
+
function clipEffectChainSteps(clip) {
|
|
450
|
+
const effects = (clip.audioEffects ?? []).filter((effect) => effect.enabled);
|
|
451
|
+
if (effects.length === 0)
|
|
452
|
+
return [];
|
|
453
|
+
const tailSec = clipAudioTailSec(effects);
|
|
454
|
+
return [
|
|
455
|
+
`aresample=${EFFECT_SAMPLE_RATE}`,
|
|
456
|
+
...(tailSec > 0 ? [`apad=pad_dur=${tailSec.toFixed(3)}`] : []),
|
|
457
|
+
...effects.flatMap(clipAudioEffectSteps)
|
|
458
|
+
];
|
|
459
|
+
}
|
|
460
|
+
/**
|
|
461
|
+
* The DSP chain of a track, in array order. Only audio and midi tracks carry
|
|
462
|
+
* one, as in the preview, where `AudioGraph.updateTracks` applies effects to
|
|
463
|
+
* those tracks alone.
|
|
464
|
+
*/
|
|
465
|
+
function trackEffectSteps(track) {
|
|
466
|
+
if (!track || (track.type !== "audio" && track.type !== "midi"))
|
|
467
|
+
return [];
|
|
468
|
+
const steps = (track.effects ?? [])
|
|
469
|
+
.filter((effect) => effect.enabled)
|
|
470
|
+
.flatMap(audioEffectSteps);
|
|
471
|
+
return steps.length > 0 ? [`aresample=${EFFECT_SAMPLE_RATE}`, ...steps] : [];
|
|
472
|
+
}
|
|
473
|
+
/**
|
|
474
|
+
* Anything that draws: media, titles, shapes, a caption riding a clip, or a 3D
|
|
475
|
+
* clip whose bake still matches it (a live one draws only in the editor).
|
|
476
|
+
*/
|
|
336
477
|
function hasRenderableVisual(seq) {
|
|
337
478
|
const tracks = trackById(seq.tracks);
|
|
338
479
|
return seq.clips.some((clip) => {
|
|
@@ -347,7 +488,14 @@ function hasRenderableVisual(seq) {
|
|
|
347
488
|
return !!clip.textStyle;
|
|
348
489
|
if (clip.mediaType === "shape")
|
|
349
490
|
return !!clip.shapeStyle;
|
|
350
|
-
|
|
491
|
+
if (clip.mediaType === "model3d") {
|
|
492
|
+
const bake = clip.model3dStyle?.bake;
|
|
493
|
+
return (bake !== undefined &&
|
|
494
|
+
computeModel3DBakeHash(clip, seq) === bake.dependencyHash);
|
|
495
|
+
}
|
|
496
|
+
return ((clip.mediaType === "video" ||
|
|
497
|
+
clip.mediaType === "image" ||
|
|
498
|
+
clip.mediaType === "overlay") &&
|
|
351
499
|
!!clip.currentAssetId);
|
|
352
500
|
});
|
|
353
501
|
}
|
|
@@ -534,10 +682,21 @@ async function renderRoughCut(opts) {
|
|
|
534
682
|
* WebM track, which is why `audioCodec` is passed rather than assumed.
|
|
535
683
|
*/
|
|
536
684
|
async function mixAudioInto(opts) {
|
|
537
|
-
const { basePath, clips, baseHasAudio, assets, midi, workDir, extension, audioCodec } = opts;
|
|
685
|
+
const { basePath, clips, tracks, baseHasAudio, assets, midi, workDir, extension, audioCodec } = opts;
|
|
538
686
|
const inputs = ["-i", basePath];
|
|
539
687
|
const filters = [];
|
|
540
688
|
const labels = [];
|
|
689
|
+
// Each clip's stream(s), grouped by track so a track's effect chain runs
|
|
690
|
+
// over the sum of its clips, as the preview's chain does.
|
|
691
|
+
const trackLabels = new Map();
|
|
692
|
+
const labelsOf = (clip) => {
|
|
693
|
+
let list = trackLabels.get(clip.trackId);
|
|
694
|
+
if (!list) {
|
|
695
|
+
list = [];
|
|
696
|
+
trackLabels.set(clip.trackId, list);
|
|
697
|
+
}
|
|
698
|
+
return list;
|
|
699
|
+
};
|
|
541
700
|
let inputIndex = 1;
|
|
542
701
|
for (const [i, clip] of clips.entries()) {
|
|
543
702
|
const audioPath = await audioSourcePath(clip, assets, midi);
|
|
@@ -554,7 +713,7 @@ async function mixAudioInto(opts) {
|
|
|
554
713
|
inputs.push("-i", audioPath);
|
|
555
714
|
const label = `a${i}`;
|
|
556
715
|
filters.push(audioClipFilter(clip, inputIndex, label));
|
|
557
|
-
|
|
716
|
+
labelsOf(clip).push(`[${label}]`);
|
|
558
717
|
inputIndex += 1;
|
|
559
718
|
continue;
|
|
560
719
|
}
|
|
@@ -577,19 +736,37 @@ async function mixAudioInto(opts) {
|
|
|
577
736
|
// So a faded remapped clip is mixed back into one stream first, at its
|
|
578
737
|
// timeline offsets, and faded there. `normalize=0` keeps the level: the
|
|
579
738
|
// stretches abut rather than overlap, so nothing is summed.
|
|
580
|
-
|
|
739
|
+
// Clip effects likewise run on the whole clip, after its fades.
|
|
740
|
+
const effectSteps = clipEffectChainSteps(clip);
|
|
741
|
+
if (segmentLabels.length > 0 &&
|
|
742
|
+
(hasClipFade(clip) || effectSteps.length > 0)) {
|
|
581
743
|
const faded = `a${i}_fade`;
|
|
744
|
+
const clipSteps = [...clipFadeSteps(clip, clip.startMs), ...effectSteps];
|
|
582
745
|
const steps = segmentLabels.length > 1
|
|
583
746
|
? [
|
|
584
747
|
`amix=inputs=${segmentLabels.length}:duration=longest:normalize=0`,
|
|
585
|
-
...
|
|
748
|
+
...clipSteps
|
|
586
749
|
]
|
|
587
|
-
:
|
|
750
|
+
: clipSteps;
|
|
588
751
|
filters.push(`${segmentLabels.join("")}${steps.join(",")}[${faded}]`);
|
|
589
|
-
|
|
752
|
+
labelsOf(clip).push(`[${faded}]`);
|
|
590
753
|
continue;
|
|
591
754
|
}
|
|
592
|
-
|
|
755
|
+
labelsOf(clip).push(...segmentLabels);
|
|
756
|
+
}
|
|
757
|
+
const byTrack = trackById(tracks);
|
|
758
|
+
for (const [trackId, streams] of trackLabels) {
|
|
759
|
+
const steps = trackEffectSteps(byTrack.get(trackId));
|
|
760
|
+
if (steps.length === 0) {
|
|
761
|
+
labels.push(...streams);
|
|
762
|
+
continue;
|
|
763
|
+
}
|
|
764
|
+
const label = `t${filters.length}`;
|
|
765
|
+
const head = streams.length > 1
|
|
766
|
+
? [`amix=inputs=${streams.length}:duration=longest:normalize=0`]
|
|
767
|
+
: [];
|
|
768
|
+
filters.push(`${streams.join("")}${[...head, ...steps].join(",")}[${label}]`);
|
|
769
|
+
labels.push(`[${label}]`);
|
|
593
770
|
}
|
|
594
771
|
if (labels.length === 0)
|
|
595
772
|
return basePath;
|
|
@@ -633,6 +810,7 @@ export async function mixCompositedTimelineAudio(opts) {
|
|
|
633
810
|
return mixAudioInto({
|
|
634
811
|
basePath,
|
|
635
812
|
clips,
|
|
813
|
+
tracks: sequence.tracks,
|
|
636
814
|
baseHasAudio: false,
|
|
637
815
|
assets: { path: resolveAssetPath },
|
|
638
816
|
midi: new MidiFiles(workDir, sequence, resolveAssetPath),
|
|
@@ -641,6 +819,13 @@ export async function mixCompositedTimelineAudio(opts) {
|
|
|
641
819
|
audioCodec: output.audioCodec ?? "aac"
|
|
642
820
|
});
|
|
643
821
|
}
|
|
822
|
+
/**
|
|
823
|
+
* Whether `RenderTimeline` tries headless Chromium first. On unless
|
|
824
|
+
* `NODETOOL_TIMELINE_RENDERER=node` asks for the Node compositor.
|
|
825
|
+
*/
|
|
826
|
+
function browserRendererEnabled() {
|
|
827
|
+
return process.env["NODETOOL_TIMELINE_RENDERER"]?.trim().toLowerCase() !== "node";
|
|
828
|
+
}
|
|
644
829
|
/** Shortest gap between two `node_progress` posts — four a second. */
|
|
645
830
|
const PROGRESS_INTERVAL_MS = 250;
|
|
646
831
|
/** Smallest draft render. Below this a frame says nothing about the cut. */
|
|
@@ -746,66 +931,92 @@ export class RenderTimelineNode extends BaseNode {
|
|
|
746
931
|
let baseHasAudio;
|
|
747
932
|
let audioToMix;
|
|
748
933
|
let renderMode;
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
934
|
+
const browserPath = path.join(workDir, `rendered.${output.extension}`);
|
|
935
|
+
const renderedInBrowser = browserRendererEnabled()
|
|
936
|
+
? await this.renderInBrowser(ctx, {
|
|
752
937
|
sequence: seq,
|
|
753
938
|
width,
|
|
754
939
|
height,
|
|
755
940
|
fps,
|
|
756
941
|
durationMs,
|
|
757
|
-
resolveAssetPath: (assetId) => assets.path(assetId),
|
|
758
|
-
outPath: basePath,
|
|
759
942
|
output,
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
this.log(ctx, `Font "${family}" could not be resolved from Google Fonts — drawn in the fallback face`, "warning");
|
|
772
|
-
}
|
|
773
|
-
renderMode = "composited";
|
|
774
|
-
// A composite carries no sound: every audible clip is mixed in below,
|
|
775
|
-
// including the audio muxed into video clips.
|
|
943
|
+
bitrate: Math.max(0, Number(this.bitrate) || 0),
|
|
944
|
+
resolveAssetPath: (assetId) => assets.path(assetId),
|
|
945
|
+
outPath: browserPath,
|
|
946
|
+
workDir
|
|
947
|
+
})
|
|
948
|
+
: false;
|
|
949
|
+
if (renderedInBrowser) {
|
|
950
|
+
basePath = browserPath;
|
|
951
|
+
renderMode = "browser";
|
|
952
|
+
// The browser writes picture only: every audible clip is mixed in
|
|
953
|
+
// below, exactly as for a Node composite.
|
|
776
954
|
baseHasAudio = false;
|
|
777
955
|
audioToMix = includeAudio
|
|
778
956
|
? [...embeddedAudioClips(seq), ...audioClips]
|
|
779
957
|
: [];
|
|
780
958
|
}
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
959
|
+
else
|
|
960
|
+
try {
|
|
961
|
+
basePath = path.join(workDir, `composited.${output.extension}`);
|
|
962
|
+
const result = await renderTimelineComposited({
|
|
963
|
+
sequence: seq,
|
|
964
|
+
width,
|
|
965
|
+
height,
|
|
966
|
+
fps,
|
|
967
|
+
durationMs,
|
|
968
|
+
resolveAssetPath: (assetId) => assets.path(assetId),
|
|
969
|
+
outPath: basePath,
|
|
970
|
+
output,
|
|
971
|
+
onProgress: this.progressReporter(ctx),
|
|
972
|
+
signal: ctx.signal
|
|
973
|
+
});
|
|
974
|
+
const { skippedClips, fontsUnavailable } = result;
|
|
975
|
+
if (result.complete === false) {
|
|
976
|
+
throw new Error(`Timeline export is incomplete: ${result.diagnostics.map((item) => `${item.clipId} (${item.startMs}–${item.endMs}ms): ${item.detail}`).join("; ")}`);
|
|
977
|
+
}
|
|
978
|
+
for (const name of skippedClips) {
|
|
979
|
+
this.log(ctx, `Clip "${name}" was skipped — its media could not be decoded`, "warning");
|
|
980
|
+
}
|
|
981
|
+
for (const family of fontsUnavailable) {
|
|
982
|
+
this.log(ctx, `Font "${family}" could not be resolved from Google Fonts — drawn in the fallback face`, "warning");
|
|
983
|
+
}
|
|
984
|
+
renderMode = "composited";
|
|
985
|
+
// A composite carries no sound: every audible clip is mixed in below,
|
|
986
|
+
// including the audio muxed into video clips.
|
|
987
|
+
baseHasAudio = false;
|
|
988
|
+
audioToMix = includeAudio
|
|
989
|
+
? [...embeddedAudioClips(seq), ...audioClips]
|
|
990
|
+
: [];
|
|
991
|
+
}
|
|
992
|
+
catch (error) {
|
|
993
|
+
if (!(error instanceof CompositorUnavailableError))
|
|
994
|
+
throw error;
|
|
995
|
+
// The rough cut is H.264 in MP4 over opaque black — it concatenates
|
|
996
|
+
// normalized segments and never touches the compositor. Handing that
|
|
997
|
+
// back for an alpha or non-MP4 request would answer a different
|
|
998
|
+
// question than the one asked, so the request fails instead.
|
|
999
|
+
if (output.format !== "mp4" || output.alpha) {
|
|
1000
|
+
throw new Error(`${error.message} The "${output.format}" format` +
|
|
1001
|
+
`${output.alpha ? " with transparency" : ""} needs the ` +
|
|
1002
|
+
"compositor; the rough-cut fallback only writes opaque MP4.");
|
|
1003
|
+
}
|
|
1004
|
+
this.log(ctx, `${error.message} Falling back to a rough cut — clip transforms, ` +
|
|
1005
|
+
"effects, transitions, captions and overlay tracks are not applied.", "warning");
|
|
1006
|
+
basePath = await renderRoughCut({
|
|
1007
|
+
seq,
|
|
1008
|
+
clips: renderableVideoClips(seq),
|
|
1009
|
+
audioClips,
|
|
1010
|
+
assets,
|
|
1011
|
+
workDir,
|
|
1012
|
+
width,
|
|
1013
|
+
height,
|
|
1014
|
+
fps
|
|
1015
|
+
});
|
|
1016
|
+
renderMode = "rough_cut";
|
|
1017
|
+
baseHasAudio = true;
|
|
1018
|
+
audioToMix = audioClips;
|
|
792
1019
|
}
|
|
793
|
-
this.log(ctx, `${error.message} Falling back to a rough cut — clip transforms, ` +
|
|
794
|
-
"effects, transitions, captions and overlay tracks are not applied.", "warning");
|
|
795
|
-
basePath = await renderRoughCut({
|
|
796
|
-
seq,
|
|
797
|
-
clips: renderableVideoClips(seq),
|
|
798
|
-
audioClips,
|
|
799
|
-
assets,
|
|
800
|
-
workDir,
|
|
801
|
-
width,
|
|
802
|
-
height,
|
|
803
|
-
fps
|
|
804
|
-
});
|
|
805
|
-
renderMode = "rough_cut";
|
|
806
|
-
baseHasAudio = true;
|
|
807
|
-
audioToMix = audioClips;
|
|
808
|
-
}
|
|
809
1020
|
// A PNG sequence is a zip of stills: there is no stream to mux a
|
|
810
1021
|
// soundtrack onto, so the audio clips are reported rather than dropped
|
|
811
1022
|
// silently.
|
|
@@ -839,6 +1050,7 @@ export class RenderTimelineNode extends BaseNode {
|
|
|
839
1050
|
? await mixAudioInto({
|
|
840
1051
|
basePath,
|
|
841
1052
|
clips: audioToMix,
|
|
1053
|
+
tracks: seq.tracks,
|
|
842
1054
|
baseHasAudio,
|
|
843
1055
|
assets,
|
|
844
1056
|
midi,
|
|
@@ -895,6 +1107,48 @@ export class RenderTimelineNode extends BaseNode {
|
|
|
895
1107
|
return;
|
|
896
1108
|
throw new MissingEncoderError(encoder, output.format);
|
|
897
1109
|
}
|
|
1110
|
+
/**
|
|
1111
|
+
* Render in headless Chromium. False when no browser could be launched,
|
|
1112
|
+
* after a warning that the Node compositor renders instead.
|
|
1113
|
+
*/
|
|
1114
|
+
async renderInBrowser(ctx, options) {
|
|
1115
|
+
try {
|
|
1116
|
+
const result = await renderTimelineInBrowser({
|
|
1117
|
+
...options,
|
|
1118
|
+
onProgress: this.progressReporter(ctx),
|
|
1119
|
+
signal: ctx.signal
|
|
1120
|
+
});
|
|
1121
|
+
const compositor = result.backend === "webgpu"
|
|
1122
|
+
? `the WebGPU compositor (${result.adapter ?? "unknown adapter"})`
|
|
1123
|
+
: result.webgpuUnusable
|
|
1124
|
+
? `the Canvas 2D compositor (WebGPU could not be read back: ${result.webgpuUnusable})`
|
|
1125
|
+
: "the Canvas 2D compositor (no WebGPU adapter)";
|
|
1126
|
+
const encoding = result.encoding.kind === "png"
|
|
1127
|
+
? "PNG frames"
|
|
1128
|
+
: `${result.encoding.codec} in ${result.encoding.container}`;
|
|
1129
|
+
this.log(ctx, `Rendered ${result.frames} frames in headless Chromium (${result.browser}) ` +
|
|
1130
|
+
`with ${compositor}, encoded as ${encoding}` +
|
|
1131
|
+
`${result.transcoded ? " and transcoded with ffmpeg" : ""}.`, "info");
|
|
1132
|
+
for (const family of result.fontsUnavailable) {
|
|
1133
|
+
this.log(ctx, `Font "${family}" could not be resolved from Google Fonts — drawn in the fallback face`, "warning");
|
|
1134
|
+
}
|
|
1135
|
+
if (result.convertedSources > 0) {
|
|
1136
|
+
this.log(ctx, `This browser cannot decode ${result.convertedSources} of the clips, so they were ` +
|
|
1137
|
+
"converted to VP9 before rendering. Google Chrome decodes them directly.", "warning");
|
|
1138
|
+
}
|
|
1139
|
+
if (result.degradedBakes > 0) {
|
|
1140
|
+
this.log(ctx, `${result.degradedBakes} baked 3D clip(s) could not be decoded and were drawn from the live 3D layer instead.`, "warning");
|
|
1141
|
+
}
|
|
1142
|
+
return true;
|
|
1143
|
+
}
|
|
1144
|
+
catch (error) {
|
|
1145
|
+
if (!(error instanceof BrowserRenderUnavailableError))
|
|
1146
|
+
throw error;
|
|
1147
|
+
this.log(ctx, `${error.message} Rendering with the Node compositor instead, which can ` +
|
|
1148
|
+
"differ from the editor preview and decodes variable-frame-rate clips more slowly.", "warning");
|
|
1149
|
+
return false;
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
898
1152
|
/** Post on the node's log channel, where the run has one. */
|
|
899
1153
|
log(context, content, severity) {
|
|
900
1154
|
if (!this.__node_id)
|