aivee 0.0.0-stage → 0.1.0-dev.7025438
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +117 -2
- package/docs/architecture.md +401 -0
- package/docs/aspect-ratio-presets.md +72 -0
- package/docs/releasing.md +19 -0
- package/docs/usage.md +1193 -0
- package/package.json +50 -4
- package/requirements-otio.txt +1 -0
- package/src/capabilities.mjs +307 -0
- package/src/cli.mjs +646 -0
- package/src/doctor.mjs +133 -0
- package/src/edits/analysis-to-edl.mjs +43 -0
- package/src/edits/detect-silence.mjs +51 -0
- package/src/edits/find-mistake-candidates.mjs +42 -0
- package/src/edits/plan-shot-placements.mjs +105 -0
- package/src/edits/plan-silence-cuts.mjs +49 -0
- package/src/edits/retime-transcript.mjs +105 -0
- package/src/files.mjs +80 -0
- package/src/index.mjs +44 -0
- package/src/interchange/otio.mjs +204 -0
- package/src/interchange/otio_bridge.py +521 -0
- package/src/media/analysis-provider.mjs +75 -0
- package/src/media/compatibility.mjs +45 -0
- package/src/media/ffmpeg-capabilities.mjs +129 -0
- package/src/media/frame-rate.mjs +58 -0
- package/src/media/frame-time.mjs +22 -0
- package/src/media/preview-assets.mjs +350 -0
- package/src/media/probe.mjs +149 -0
- package/src/media/render-cuts.mjs +90 -0
- package/src/media/scene-detection.mjs +107 -0
- package/src/media/scene-review.mjs +205 -0
- package/src/project/assets.mjs +1194 -0
- package/src/project/create.mjs +100 -0
- package/src/project/presets.mjs +44 -0
- package/src/render/checkpoints.mjs +192 -0
- package/src/render/formats.mjs +163 -0
- package/src/render/hyperframes.mjs +590 -0
- package/src/render/process.mjs +111 -0
- package/src/render/progress.mjs +46 -0
- package/src/render/timeline-control.mjs +53 -0
- package/src/timeline/assemble.mjs +119 -0
- package/src/timeline/audio-effects.mjs +75 -0
- package/src/timeline/audio-mastering.mjs +236 -0
- package/src/timeline/audio-mix.mjs +105 -0
- package/src/timeline/audio-track-effects.mjs +139 -0
- package/src/timeline/audio-volume.mjs +63 -0
- package/src/timeline/background.mjs +8 -0
- package/src/timeline/blend-mode.mjs +34 -0
- package/src/timeline/caption-layouts.mjs +61 -0
- package/src/timeline/captions.mjs +80 -0
- package/src/timeline/edit.mjs +965 -0
- package/src/timeline/fingerprint.mjs +6 -0
- package/src/timeline/from-edl.mjs +31 -0
- package/src/timeline/history-store.mjs +281 -0
- package/src/timeline/history.mjs +69 -0
- package/src/timeline/inspect.mjs +117 -0
- package/src/timeline/keyframes.mjs +158 -0
- package/src/timeline/layers.mjs +18 -0
- package/src/timeline/mask.mjs +53 -0
- package/src/timeline/project.mjs +52 -0
- package/src/timeline/render.mjs +759 -0
- package/src/timeline/sidechain-render.mjs +209 -0
- package/src/timeline/source-fit.mjs +15 -0
- package/src/timeline/speech-bubble.mjs +26 -0
- package/src/timeline/speed-ramp.mjs +188 -0
- package/src/timeline/track-state.mjs +22 -0
- package/src/timeline/transition.mjs +34 -0
- package/src/timeline/validate.mjs +322 -0
- package/src/transcript/corrections.mjs +21 -0
- package/src/transcript/normalize.mjs +17 -0
- package/src/transcript/provider.mjs +42 -0
- package/src/transcript/subtitles.mjs +57 -0
- package/src/transcript/validate.mjs +39 -0
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { sampleAudioVolumeKeyframes } from './audio-volume.mjs';
|
|
2
|
+
|
|
3
|
+
export function validateAudioMix(timeline) {
|
|
4
|
+
const mix = timeline.audioMix;
|
|
5
|
+
if (mix === undefined) return timeline;
|
|
6
|
+
if (!mix || typeof mix !== 'object' || Array.isArray(mix) || (mix.ducking !== undefined && !Array.isArray(mix.ducking))
|
|
7
|
+
|| (mix.sidechain !== undefined && !Array.isArray(mix.sidechain))) {
|
|
8
|
+
throw new TypeError('Timeline audioMix ducking and sidechain settings must be arrays when provided.');
|
|
9
|
+
}
|
|
10
|
+
const tracks = new Map(timeline.tracks.map((track) => [track.id, track]));
|
|
11
|
+
for (const [index, rule] of (mix.ducking ?? []).entries()) {
|
|
12
|
+
const source = tracks.get(rule?.sourceTrackId);
|
|
13
|
+
const target = tracks.get(rule?.targetTrackId);
|
|
14
|
+
if (!source || !target || source.kind !== 'audio' || target.kind !== 'audio' || source === target) throw new TypeError(`Audio ducking rule ${index + 1} must reference two different audio tracks.`);
|
|
15
|
+
if (!Number.isFinite(rule.attenuation) || rule.attenuation < 0 || rule.attenuation > 1) throw new TypeError(`Audio ducking rule ${index + 1} attenuation must be between 0 and 1.`);
|
|
16
|
+
for (const field of ['attackSeconds', 'releaseSeconds']) if (!Number.isFinite(rule[field] ?? 0) || (rule[field] ?? 0) < 0) throw new TypeError(`Audio ducking rule ${index + 1} ${field} must be non-negative.`);
|
|
17
|
+
}
|
|
18
|
+
for (const [index, rule] of (mix.sidechain ?? []).entries()) {
|
|
19
|
+
const source = tracks.get(rule?.sourceTrackId);
|
|
20
|
+
const target = tracks.get(rule?.targetTrackId);
|
|
21
|
+
if (!source || !target || source.kind !== 'audio' || target.kind !== 'audio' || source === target
|
|
22
|
+
|| source.clips.length === 0 || target.clips.length === 0) {
|
|
23
|
+
throw new TypeError(`Audio sidechain rule ${index + 1} must reference two different non-empty audio tracks.`);
|
|
24
|
+
}
|
|
25
|
+
const settings = resolveAudioSidechainSettings(rule);
|
|
26
|
+
if (!Number.isFinite(settings.thresholdDb) || settings.thresholdDb < -60 || settings.thresholdDb > 0) throw new TypeError(`Audio sidechain rule ${index + 1} thresholdDb must be between -60 and 0.`);
|
|
27
|
+
if (!Number.isFinite(settings.ratio) || settings.ratio < 1 || settings.ratio > 20) throw new TypeError(`Audio sidechain rule ${index + 1} ratio must be between 1 and 20.`);
|
|
28
|
+
if (!Number.isFinite(settings.attackSeconds) || settings.attackSeconds < 0.00001 || settings.attackSeconds > 2) throw new TypeError(`Audio sidechain rule ${index + 1} attackSeconds must be between 0.00001 and 2.`);
|
|
29
|
+
if (!Number.isFinite(settings.releaseSeconds) || settings.releaseSeconds < 0.00001 || settings.releaseSeconds > 9) throw new TypeError(`Audio sidechain rule ${index + 1} releaseSeconds must be between 0.00001 and 9.`);
|
|
30
|
+
for (const field of Object.keys(rule)) {
|
|
31
|
+
if (!['sourceTrackId', 'targetTrackId', 'thresholdDb', 'ratio', 'attackSeconds', 'releaseSeconds'].includes(field)) {
|
|
32
|
+
throw new TypeError(`Audio sidechain rule ${index + 1} has an unsupported field: ${field}.`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return timeline;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function resolveAudioSidechainSettings(rule) {
|
|
40
|
+
return {
|
|
41
|
+
thresholdDb: rule.thresholdDb ?? -24,
|
|
42
|
+
ratio: rule.ratio ?? 4,
|
|
43
|
+
attackSeconds: rule.attackSeconds ?? 0.01,
|
|
44
|
+
releaseSeconds: rule.releaseSeconds ?? 0.25,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export function buildAudioVolumeAutomation(timeline, track, clip) {
|
|
49
|
+
const start = clip.timelineStart;
|
|
50
|
+
const end = start + clip.durationSeconds;
|
|
51
|
+
const baseVolume = (clip.volume ?? 1) * (track.volume ?? 1);
|
|
52
|
+
const duckRamps = [];
|
|
53
|
+
for (const rule of timeline.audioMix?.ducking ?? []) {
|
|
54
|
+
if (rule.targetTrackId !== track.id) continue;
|
|
55
|
+
const source = timeline.tracks.find((item) => item.id === rule.sourceTrackId);
|
|
56
|
+
for (const sourceClip of source._duckingIntervals ?? source.clips) {
|
|
57
|
+
const duckStart = Math.max(start, sourceClip.timelineStart);
|
|
58
|
+
const duckEnd = Math.min(end, sourceClip.timelineStart + sourceClip.durationSeconds);
|
|
59
|
+
if (duckEnd <= duckStart) continue;
|
|
60
|
+
duckRamps.push({
|
|
61
|
+
start: duckStart,
|
|
62
|
+
end: duckEnd,
|
|
63
|
+
attenuation: rule.attenuation,
|
|
64
|
+
attackSeconds: rule.attackSeconds ?? 0,
|
|
65
|
+
releaseSeconds: rule.releaseSeconds ?? 0,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const points = new Set([start, end]);
|
|
71
|
+
const fadeIn = clip.fadeInSeconds ?? 0;
|
|
72
|
+
const fadeOut = clip.fadeOutSeconds ?? 0;
|
|
73
|
+
if (fadeIn > 0) points.add(start + fadeIn);
|
|
74
|
+
if (fadeOut > 0) points.add(end - fadeOut);
|
|
75
|
+
for (const keyframe of clip.volumeKeyframes ?? []) points.add(start + keyframe.time);
|
|
76
|
+
for (const keyframe of track.volumeKeyframes ?? []) {
|
|
77
|
+
if (keyframe.time > start && keyframe.time < end) points.add(keyframe.time);
|
|
78
|
+
}
|
|
79
|
+
for (const ramp of duckRamps) {
|
|
80
|
+
points.add(Math.max(start, ramp.start - ramp.attackSeconds));
|
|
81
|
+
points.add(ramp.start);
|
|
82
|
+
points.add(ramp.end);
|
|
83
|
+
points.add(Math.min(end, ramp.end + ramp.releaseSeconds));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return [...points].sort((a, b) => a - b).map((time) => {
|
|
87
|
+
let volume = baseVolume;
|
|
88
|
+
volume *= sampleAudioVolumeKeyframes(track.volumeKeyframes, time);
|
|
89
|
+
volume *= sampleAudioVolumeKeyframes(clip.volumeKeyframes, time - start);
|
|
90
|
+
if (fadeIn > 0 && time < start + fadeIn) volume *= Math.max(0, (time - start) / fadeIn);
|
|
91
|
+
if (fadeOut > 0 && time > end - fadeOut) volume *= Math.max(0, (end - time) / fadeOut);
|
|
92
|
+
for (const ramp of duckRamps) volume *= duckFactorAt(time, ramp);
|
|
93
|
+
return { time, volume: Number(volume.toFixed(6)) };
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function duckFactorAt(time, ramp) {
|
|
98
|
+
if (time < ramp.start) {
|
|
99
|
+
const attackStart = ramp.start - ramp.attackSeconds;
|
|
100
|
+
if (ramp.attackSeconds === 0 || time <= attackStart) return 1;
|
|
101
|
+
return 1 - ((time - attackStart) / ramp.attackSeconds) * (1 - ramp.attenuation);
|
|
102
|
+
}
|
|
103
|
+
if (time <= ramp.end || ramp.releaseSeconds === 0 || time >= ramp.end + ramp.releaseSeconds) return time <= ramp.end ? ramp.attenuation : 1;
|
|
104
|
+
return ramp.attenuation + ((time - ramp.end) / ramp.releaseSeconds) * (1 - ramp.attenuation);
|
|
105
|
+
}
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { spawnSync } from 'node:child_process';
|
|
3
|
+
import { rmSync } from 'node:fs';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { runProcess } from '../render/process.mjs';
|
|
6
|
+
import { buildAudioClipEffectFilters } from './audio-effects.mjs';
|
|
7
|
+
import { buildAudioVolumeAutomation } from './audio-mix.mjs';
|
|
8
|
+
|
|
9
|
+
export function applyAudioTrackEffectProxies(root, timeline, temporaryAssets) {
|
|
10
|
+
const context = prepareAudioTrackEffectProxies(root, timeline, temporaryAssets);
|
|
11
|
+
try {
|
|
12
|
+
for (const job of context.jobs) {
|
|
13
|
+
const result = spawnSync('ffmpeg', job.args, {
|
|
14
|
+
encoding: 'utf8',
|
|
15
|
+
windowsHide: true,
|
|
16
|
+
cwd: root,
|
|
17
|
+
maxBuffer: 4 * 1024 * 1024,
|
|
18
|
+
});
|
|
19
|
+
if (result.error || result.status !== 0) {
|
|
20
|
+
throw new Error(`Could not apply audio effects to track ${job.trackId}: ${result.error?.message ?? result.stderr.trim()}`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return context.timeline;
|
|
24
|
+
} catch (error) {
|
|
25
|
+
removePreparedAssets(temporaryAssets, context.startIndex);
|
|
26
|
+
throw error;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export async function applyAudioTrackEffectProxiesAsync(root, timeline, temporaryAssets, renderControl) {
|
|
31
|
+
const context = prepareAudioTrackEffectProxies(root, timeline, temporaryAssets);
|
|
32
|
+
try {
|
|
33
|
+
for (const job of context.jobs) {
|
|
34
|
+
renderControl.assertActive();
|
|
35
|
+
try {
|
|
36
|
+
await runProcess('ffmpeg', job.args, { cwd: root, signal: renderControl.signal, stdio: 'capture' });
|
|
37
|
+
} catch (error) {
|
|
38
|
+
if (renderControl.signal.aborted) throw renderControl.signal.reason ?? error;
|
|
39
|
+
throw new Error(`Could not apply audio effects to track ${job.trackId}: ${error.message}`, { cause: error });
|
|
40
|
+
}
|
|
41
|
+
renderControl.assertActive();
|
|
42
|
+
}
|
|
43
|
+
return context.timeline;
|
|
44
|
+
} catch (error) {
|
|
45
|
+
removePreparedAssets(temporaryAssets, context.startIndex);
|
|
46
|
+
throw error;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function prepareAudioTrackEffectProxies(root, timeline, temporaryAssets) {
|
|
51
|
+
const next = structuredClone(timeline);
|
|
52
|
+
const startIndex = temporaryAssets.length;
|
|
53
|
+
const jobs = [];
|
|
54
|
+
try {
|
|
55
|
+
for (const track of next.tracks) {
|
|
56
|
+
if (track.kind !== 'audio' || !track.audioEffects?.length || track.clips.length === 0) continue;
|
|
57
|
+
const sourceClips = track.clips.map((clip) => ({ timelineStart: clip.timelineStart, durationSeconds: clip.durationSeconds }));
|
|
58
|
+
const labels = [];
|
|
59
|
+
const filters = [];
|
|
60
|
+
const args = [];
|
|
61
|
+
for (const [index, clip] of track.clips.entries()) {
|
|
62
|
+
const asset = next.assets.find((entry) => entry.id === clip.assetId);
|
|
63
|
+
if (!asset) throw new TypeError(`Track ${track.id} clip ${clip.id} is missing its audio asset.`);
|
|
64
|
+
args.push('-i', join(root, ...asset.path.split('/')));
|
|
65
|
+
const automation = buildAudioVolumeAutomation(next, track, clip);
|
|
66
|
+
const label = `trackclip${index}`;
|
|
67
|
+
labels.push(`[${label}]`);
|
|
68
|
+
const steps = [
|
|
69
|
+
`atrim=start=${clip.sourceIn}:end=${clip.sourceOut}`,
|
|
70
|
+
'asetpts=PTS-STARTPTS',
|
|
71
|
+
'aresample=48000',
|
|
72
|
+
'aformat=sample_fmts=fltp:channel_layouts=stereo',
|
|
73
|
+
`volume='${toVolumeExpression(automation, clip.timelineStart)}':eval=frame`,
|
|
74
|
+
`adelay=delays=${Math.round(clip.timelineStart * 1000)}:all=1`,
|
|
75
|
+
];
|
|
76
|
+
filters.push(`[${index}:a:${clip.audioStreamIndex ?? 0}]${steps.join(',')}[${label}]`);
|
|
77
|
+
}
|
|
78
|
+
const effectFilters = buildAudioClipEffectFilters(track.audioEffects);
|
|
79
|
+
const outputLabel = 'trackfxout';
|
|
80
|
+
filters.push(`${labels.join('')}amix=inputs=${labels.length}:duration=longest:normalize=0,apad=whole_dur=${next.project.durationSeconds},atrim=duration=${next.project.durationSeconds},asetpts=PTS-STARTPTS${effectFilters.length ? `,${effectFilters.join(',')}` : ''}[${outputLabel}]`);
|
|
81
|
+
const id = `track-effects-${track.id}-${randomUUID().slice(0, 8)}`;
|
|
82
|
+
const relativePath = `assets/.${id}-${randomUUID()}.wav`;
|
|
83
|
+
const absolutePath = join(root, ...relativePath.split('/'));
|
|
84
|
+
temporaryAssets.push(absolutePath);
|
|
85
|
+
jobs.push({
|
|
86
|
+
trackId: track.id,
|
|
87
|
+
args: [
|
|
88
|
+
'-nostdin', '-hide_banner', '-v', 'error', '-y', '-filter_threads', '1', '-filter_complex_threads', '1',
|
|
89
|
+
...args, '-filter_complex', filters.join(';'), '-map', `[${outputLabel}]`, '-vn', '-ar', '48000', '-ac', '2', '-c:a', 'pcm_s16le', absolutePath,
|
|
90
|
+
],
|
|
91
|
+
});
|
|
92
|
+
next.assets.push({ id, path: relativePath, type: 'audio', hasAudio: true, durationSeconds: next.project.durationSeconds });
|
|
93
|
+
track.clips = [{
|
|
94
|
+
id: `processed-${track.id}`,
|
|
95
|
+
assetId: id,
|
|
96
|
+
sourceIn: 0,
|
|
97
|
+
sourceOut: next.project.durationSeconds,
|
|
98
|
+
timelineStart: 0,
|
|
99
|
+
durationSeconds: next.project.durationSeconds,
|
|
100
|
+
}];
|
|
101
|
+
track.volume = 1;
|
|
102
|
+
delete track.volumeKeyframes;
|
|
103
|
+
delete track.audioEffects;
|
|
104
|
+
track._duckingIntervals = sourceClips;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const effectedTrackIds = new Set(next.tracks.filter((track) => track.kind === 'audio' && track._duckingIntervals).map((track) => track.id));
|
|
108
|
+
if (next.audioMix?.ducking) {
|
|
109
|
+
const ducking = next.audioMix.ducking.filter((rule) => !effectedTrackIds.has(rule.targetTrackId));
|
|
110
|
+
if (ducking.length === 0) delete next.audioMix.ducking;
|
|
111
|
+
else next.audioMix.ducking = ducking;
|
|
112
|
+
if (Object.keys(next.audioMix).length === 0) delete next.audioMix;
|
|
113
|
+
}
|
|
114
|
+
return { timeline: next, jobs, startIndex };
|
|
115
|
+
} catch (error) {
|
|
116
|
+
removePreparedAssets(temporaryAssets, startIndex);
|
|
117
|
+
throw error;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function removePreparedAssets(temporaryAssets, startIndex) {
|
|
122
|
+
for (const asset of temporaryAssets.splice(startIndex)) rmSync(asset, { force: true });
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function toVolumeExpression(points, clipStart) {
|
|
126
|
+
if (points.length === 0) return '1';
|
|
127
|
+
let expression = String(points.at(-1).volume);
|
|
128
|
+
for (let index = points.length - 2; index >= 0; index--) {
|
|
129
|
+
const from = points[index];
|
|
130
|
+
const to = points[index + 1];
|
|
131
|
+
const fromTime = from.time - clipStart;
|
|
132
|
+
const toTime = to.time - clipStart;
|
|
133
|
+
const span = toTime - fromTime;
|
|
134
|
+
if (span <= 0) continue;
|
|
135
|
+
const linear = `(${from.volume}+(${to.volume - from.volume})*(t-${fromTime})/${span})`;
|
|
136
|
+
expression = `if(lt(t\\,${toTime})\\,${linear}\\,${expression})`;
|
|
137
|
+
}
|
|
138
|
+
return expression;
|
|
139
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
export function validateAudioVolumeKeyframes(keyframes, clip) {
|
|
2
|
+
if (keyframes === undefined) return;
|
|
3
|
+
if (!Array.isArray(keyframes) || keyframes.length === 0 || keyframes.length > 512) {
|
|
4
|
+
throw new TypeError(`Clip ${clip.id} volumeKeyframes must be an array with 1 to 512 entries.`);
|
|
5
|
+
}
|
|
6
|
+
if (keyframes[0]?.time !== 0) throw new TypeError(`Clip ${clip.id} volumeKeyframes must start at time 0.`);
|
|
7
|
+
let previousTime = -1;
|
|
8
|
+
for (const [index, keyframe] of keyframes.entries()) {
|
|
9
|
+
if (!keyframe || typeof keyframe !== 'object' || Array.isArray(keyframe)
|
|
10
|
+
|| Object.keys(keyframe).some((field) => !['time', 'volume'].includes(field))
|
|
11
|
+
|| !Number.isFinite(keyframe.time) || keyframe.time < 0 || keyframe.time > clip.durationSeconds
|
|
12
|
+
|| keyframe.time <= previousTime
|
|
13
|
+
|| !Number.isFinite(keyframe.volume) || keyframe.volume < 0 || keyframe.volume > 1) {
|
|
14
|
+
throw new TypeError(`Clip ${clip.id} volumeKeyframes entry ${index + 1} must have an increasing time and volume from 0 to 1 within the clip.`);
|
|
15
|
+
}
|
|
16
|
+
previousTime = keyframe.time;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function sampleAudioVolumeKeyframes(keyframes = [], time) {
|
|
21
|
+
if (keyframes.length === 0) return 1;
|
|
22
|
+
if (time <= keyframes[0].time) return keyframes[0].volume;
|
|
23
|
+
for (let index = 1; index < keyframes.length; index++) {
|
|
24
|
+
const next = keyframes[index];
|
|
25
|
+
if (time > next.time) continue;
|
|
26
|
+
const previous = keyframes[index - 1];
|
|
27
|
+
const amount = (time - previous.time) / (next.time - previous.time);
|
|
28
|
+
return previous.volume + (next.volume - previous.volume) * amount;
|
|
29
|
+
}
|
|
30
|
+
return keyframes.at(-1).volume;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function validateAudioTrackVolumeKeyframes(keyframes, track, projectDurationSeconds) {
|
|
34
|
+
if (keyframes === undefined) return;
|
|
35
|
+
if (track.kind !== 'audio') throw new TypeError(`Track ${track.id} volumeKeyframes are supported only on audio tracks.`);
|
|
36
|
+
if (!Array.isArray(keyframes) || keyframes.length === 0 || keyframes.length > 512) {
|
|
37
|
+
throw new TypeError(`Track ${track.id} volumeKeyframes must be an array with 1 to 512 entries.`);
|
|
38
|
+
}
|
|
39
|
+
if (keyframes[0]?.time !== 0) throw new TypeError(`Track ${track.id} volumeKeyframes must start at time 0.`);
|
|
40
|
+
let previousTime = -1;
|
|
41
|
+
for (const [index, keyframe] of keyframes.entries()) {
|
|
42
|
+
if (!keyframe || typeof keyframe !== 'object' || Array.isArray(keyframe)
|
|
43
|
+
|| Object.keys(keyframe).some((field) => !['time', 'volume'].includes(field))
|
|
44
|
+
|| !Number.isFinite(keyframe.time) || keyframe.time < 0 || keyframe.time > projectDurationSeconds
|
|
45
|
+
|| keyframe.time <= previousTime
|
|
46
|
+
|| !Number.isFinite(keyframe.volume) || keyframe.volume < 0 || keyframe.volume > 1) {
|
|
47
|
+
throw new TypeError(`Track ${track.id} volumeKeyframes entry ${index + 1} must have an increasing project time and volume from 0 to 1.`);
|
|
48
|
+
}
|
|
49
|
+
previousTime = keyframe.time;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function sliceAudioVolumeKeyframes(keyframes, start, end) {
|
|
54
|
+
if (!keyframes) return undefined;
|
|
55
|
+
const sliced = [{ time: 0, volume: sampleAudioVolumeKeyframes(keyframes, start) }];
|
|
56
|
+
for (const keyframe of keyframes) {
|
|
57
|
+
if (keyframe.time > start && keyframe.time < end) {
|
|
58
|
+
sliced.push({ time: keyframe.time - start, volume: keyframe.volume });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
if (end > start) sliced.push({ time: end - start, volume: sampleAudioVolumeKeyframes(keyframes, end) });
|
|
62
|
+
return sliced;
|
|
63
|
+
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export const DEFAULT_PROJECT_BACKGROUND_COLOR = '#10141c';
|
|
2
|
+
|
|
3
|
+
export function validateProjectBackgroundColor(value, label = 'Project backgroundColor') {
|
|
4
|
+
if (value !== null && (typeof value !== 'string' || !/^#(?:[\da-f]{3,4}|[\da-f]{6}|[\da-f]{8})$/i.test(value))) {
|
|
5
|
+
throw new TypeError(`${label} must be a 3, 4, 6, or 8 digit hex color, or null for transparent.`);
|
|
6
|
+
}
|
|
7
|
+
return value;
|
|
8
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
const BLEND_MODES = new Set([
|
|
2
|
+
'normal',
|
|
3
|
+
'multiply',
|
|
4
|
+
'screen',
|
|
5
|
+
'overlay',
|
|
6
|
+
'darken',
|
|
7
|
+
'lighten',
|
|
8
|
+
'color-dodge',
|
|
9
|
+
'color-burn',
|
|
10
|
+
'hard-light',
|
|
11
|
+
'soft-light',
|
|
12
|
+
'difference',
|
|
13
|
+
'exclusion',
|
|
14
|
+
'hue',
|
|
15
|
+
'saturation',
|
|
16
|
+
'color',
|
|
17
|
+
'luminosity',
|
|
18
|
+
]);
|
|
19
|
+
|
|
20
|
+
export function validateBlendMode(clip, trackKind) {
|
|
21
|
+
if (clip.blendMode === undefined) return;
|
|
22
|
+
if (trackKind !== 'video' && trackKind !== 'overlay') {
|
|
23
|
+
throw new TypeError(`Clip ${clip.id} blendMode is supported only on video and overlay tracks.`);
|
|
24
|
+
}
|
|
25
|
+
if (typeof clip.blendMode !== 'string' || !BLEND_MODES.has(clip.blendMode)) {
|
|
26
|
+
throw new TypeError(`Clip ${clip.id} blendMode must be one of: ${[...BLEND_MODES].join(', ')}.`);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function blendModeStyle(clip) {
|
|
31
|
+
return clip.blendMode && clip.blendMode !== 'normal'
|
|
32
|
+
? `mix-blend-mode:${clip.blendMode};`
|
|
33
|
+
: '';
|
|
34
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
const SOCIAL_LAYOUT = {
|
|
2
|
+
x: 0.08, y: 0.68, width: 0.84, height: 0.24, fontSize: 64,
|
|
3
|
+
fontWeight: 800, lineHeight: 1.05, verticalAlign: 'bottom', paddingPixels: 14,
|
|
4
|
+
wordWrap: true,
|
|
5
|
+
textShadow: { color: '#000000cc', blurPixels: 6, offsetXPixels: 0, offsetYPixels: 3 },
|
|
6
|
+
color: '#ffffff', backgroundColor: '#00000000', textAlign: 'center',
|
|
7
|
+
};
|
|
8
|
+
|
|
9
|
+
const LAYOUTS = {
|
|
10
|
+
subtitle: {
|
|
11
|
+
x: 0.08, y: 0.76, width: 0.84, height: 0.16, fontSize: 54,
|
|
12
|
+
fontWeight: 600, verticalAlign: 'bottom', paddingPixels: 12, wordWrap: true,
|
|
13
|
+
textShadow: { color: '#000000b8', blurPixels: 4, offsetXPixels: 0, offsetYPixels: 2 },
|
|
14
|
+
color: '#ffffff', backgroundColor: '#000000b8', textAlign: 'center',
|
|
15
|
+
},
|
|
16
|
+
social: SOCIAL_LAYOUT,
|
|
17
|
+
socialPop: {
|
|
18
|
+
...SOCIAL_LAYOUT,
|
|
19
|
+
wordHighlight: true, wordHighlightColor: '#ffe45c', wordHighlightAnimation: 'pop',
|
|
20
|
+
},
|
|
21
|
+
socialStack: {
|
|
22
|
+
...SOCIAL_LAYOUT,
|
|
23
|
+
y: 0.64, height: 0.3, fontSize: 58, lineHeight: 1.02,
|
|
24
|
+
wordsPerCaption: 6, wordsPerLine: 3,
|
|
25
|
+
wordHighlight: true, wordHighlightColor: '#ffe45c', wordHighlightAnimation: 'pop',
|
|
26
|
+
},
|
|
27
|
+
speechBubble: {
|
|
28
|
+
x: 0.32, y: 0.66, width: 0.62, height: 0.1, fontSize: 44,
|
|
29
|
+
fontWeight: 600, lineHeight: 1.12, verticalAlign: 'center', paddingPixels: 22,
|
|
30
|
+
wordWrap: true, wordsPerCaption: 6, wordsPerLine: 3,
|
|
31
|
+
wordHighlight: true, wordHighlightColor: '#426BEB', wordHighlightAnimation: 'scale',
|
|
32
|
+
color: '#172236', backgroundColor: '#ffffff', textAlign: 'left',
|
|
33
|
+
textShadow: { color: '#17223622', blurPixels: 10, offsetXPixels: 0, offsetYPixels: 4 },
|
|
34
|
+
bubble: { tailSide: 'bottom', tailPosition: 0.16, tailSizePixels: 26, cornerRadiusPixels: 28 },
|
|
35
|
+
},
|
|
36
|
+
lowerThird: {
|
|
37
|
+
x: 0.08, y: 0.72, width: 0.76, height: 0.18, fontSize: 48,
|
|
38
|
+
fontWeight: 600, verticalAlign: 'bottom', paddingPixels: 12, wordWrap: true,
|
|
39
|
+
textShadow: { color: '#000000b8', blurPixels: 3, offsetXPixels: 0, offsetYPixels: 2 },
|
|
40
|
+
color: '#ffffff', backgroundColor: '#000000b8', textAlign: 'left',
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
export const CAPTION_LAYOUT_PRESETS = Object.freeze(Object.fromEntries(
|
|
45
|
+
Object.entries(LAYOUTS).map(([name, layout]) => [name, Object.freeze({
|
|
46
|
+
...layout,
|
|
47
|
+
...(layout.textOutline ? { textOutline: Object.freeze({ ...layout.textOutline }) } : {}),
|
|
48
|
+
...(layout.textShadow ? { textShadow: Object.freeze({ ...layout.textShadow }) } : {}),
|
|
49
|
+
})]),
|
|
50
|
+
));
|
|
51
|
+
|
|
52
|
+
export function listCaptionLayoutPresets() {
|
|
53
|
+
return Object.keys(CAPTION_LAYOUT_PRESETS);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function resolveCaptionLayoutPreset(name = 'subtitle') {
|
|
57
|
+
if (typeof name !== 'string' || !Object.hasOwn(CAPTION_LAYOUT_PRESETS, name)) {
|
|
58
|
+
throw new TypeError(`Unknown caption layout preset: ${name}`);
|
|
59
|
+
}
|
|
60
|
+
return structuredClone(CAPTION_LAYOUT_PRESETS[name]);
|
|
61
|
+
}
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { validateTranscript } from '../transcript/validate.mjs';
|
|
2
|
+
import { resolveCaptionLayoutPreset } from './caption-layouts.mjs';
|
|
3
|
+
import { resolveSpeechBubble } from './speech-bubble.mjs';
|
|
4
|
+
|
|
5
|
+
export function createCaptionTrack(transcript, options = {}) {
|
|
6
|
+
validateTranscript(transcript);
|
|
7
|
+
const layout = resolveCaptionLayoutPreset(options.layoutPreset);
|
|
8
|
+
const wordHighlight = options.wordHighlight ?? layout.wordHighlight ?? false;
|
|
9
|
+
if (typeof wordHighlight !== 'boolean') throw new TypeError('wordHighlight must be a boolean.');
|
|
10
|
+
if (wordHighlight && transcript.granularity === 'cue') throw new TypeError('Word highlighting requires word-level transcript timing.');
|
|
11
|
+
const wordHighlightColor = options.wordHighlightColor ?? layout.wordHighlightColor ?? '#ffe45c';
|
|
12
|
+
if (typeof wordHighlightColor !== 'string' || !/^#(?:[\da-f]{3}|[\da-f]{4}|[\da-f]{6}|[\da-f]{8})$/i.test(wordHighlightColor)) {
|
|
13
|
+
throw new TypeError('wordHighlightColor must be a hex color.');
|
|
14
|
+
}
|
|
15
|
+
if (!wordHighlight && options.wordHighlightColor !== undefined) throw new TypeError('wordHighlightColor requires wordHighlight: true.');
|
|
16
|
+
const wordHighlightAnimation = options.wordHighlightAnimation ?? layout.wordHighlightAnimation ?? 'scale';
|
|
17
|
+
if (!['scale', 'pop'].includes(wordHighlightAnimation)) throw new TypeError('wordHighlightAnimation must be scale or pop.');
|
|
18
|
+
if (!wordHighlight && options.wordHighlightAnimation !== undefined) throw new TypeError('wordHighlightAnimation requires wordHighlight: true.');
|
|
19
|
+
const wordsPerCaption = options.wordsPerCaption ?? layout.wordsPerCaption ?? 6;
|
|
20
|
+
if (!Number.isSafeInteger(wordsPerCaption) || wordsPerCaption < 1 || wordsPerCaption > 20) throw new TypeError('wordsPerCaption must be an integer from 1 to 20.');
|
|
21
|
+
const wordsPerLine = options.wordsPerLine ?? layout.wordsPerLine;
|
|
22
|
+
if (wordsPerLine !== undefined && (!Number.isSafeInteger(wordsPerLine) || wordsPerLine < 1 || wordsPerLine > 20)) {
|
|
23
|
+
throw new TypeError('wordsPerLine must be an integer from 1 to 20.');
|
|
24
|
+
}
|
|
25
|
+
const wordWrap = options.wordWrap ?? layout.wordWrap ?? false;
|
|
26
|
+
if (typeof wordWrap !== 'boolean') throw new TypeError('wordWrap must be a boolean.');
|
|
27
|
+
const maxGapSeconds = options.maxGapSeconds ?? 0.8;
|
|
28
|
+
if (!Number.isFinite(maxGapSeconds) || maxGapSeconds < 0) throw new TypeError('maxGapSeconds must be a non-negative number.');
|
|
29
|
+
const bubbleSource = options.bubble ?? layout.bubble;
|
|
30
|
+
const bubble = bubbleSource === undefined ? undefined : resolveSpeechBubble(bubbleSource, layout.bubble);
|
|
31
|
+
|
|
32
|
+
const groups = [];
|
|
33
|
+
let group = [];
|
|
34
|
+
for (const word of transcript.words) {
|
|
35
|
+
if (group.length && (group.length >= wordsPerCaption || word.start - group.at(-1).end > maxGapSeconds || /[.!?]$/.test(group.at(-1).text))) {
|
|
36
|
+
groups.push(group);
|
|
37
|
+
group = [];
|
|
38
|
+
}
|
|
39
|
+
group.push(word);
|
|
40
|
+
}
|
|
41
|
+
if (group.length) groups.push(group);
|
|
42
|
+
|
|
43
|
+
const clips = groups.map((words, index) => ({
|
|
44
|
+
id: `caption-${String(index + 1).padStart(3, '0')}`,
|
|
45
|
+
timelineStart: words[0].start,
|
|
46
|
+
durationSeconds: words.at(-1).end - words[0].start,
|
|
47
|
+
text: words.map(({ text }) => text).join(' '),
|
|
48
|
+
...(wordsPerLine !== undefined ? { wordsPerLine } : {}),
|
|
49
|
+
...(wordHighlight ? {
|
|
50
|
+
wordTimings: words.map((word) => ({
|
|
51
|
+
text: word.text,
|
|
52
|
+
start: word.start - words[0].start,
|
|
53
|
+
end: word.end - words[0].start,
|
|
54
|
+
})),
|
|
55
|
+
wordHighlightColor,
|
|
56
|
+
wordHighlightAnimation,
|
|
57
|
+
} : {}),
|
|
58
|
+
x: options.x ?? layout.x,
|
|
59
|
+
y: options.y ?? layout.y,
|
|
60
|
+
width: options.width ?? layout.width,
|
|
61
|
+
height: options.height ?? layout.height,
|
|
62
|
+
fontSize: options.fontSize ?? layout.fontSize,
|
|
63
|
+
...(options.fontFamily !== undefined ? { fontFamily: options.fontFamily } : {}),
|
|
64
|
+
fontWeight: options.fontWeight ?? layout.fontWeight,
|
|
65
|
+
...(options.lineHeight !== undefined || layout.lineHeight !== undefined ? { lineHeight: options.lineHeight ?? layout.lineHeight } : {}),
|
|
66
|
+
verticalAlign: options.verticalAlign ?? layout.verticalAlign,
|
|
67
|
+
wordWrap,
|
|
68
|
+
paddingPixels: options.paddingPixels ?? layout.paddingPixels,
|
|
69
|
+
textOutline: options.textOutline ?? layout.textOutline,
|
|
70
|
+
textShadow: options.textShadow ?? layout.textShadow,
|
|
71
|
+
color: options.color ?? layout.color,
|
|
72
|
+
backgroundColor: options.backgroundColor ?? layout.backgroundColor,
|
|
73
|
+
textAlign: options.textAlign ?? layout.textAlign,
|
|
74
|
+
...(bubble === undefined ? {} : { bubble: structuredClone(bubble) }),
|
|
75
|
+
...(options.fadeInSeconds ? { fadeInSeconds: options.fadeInSeconds } : {}),
|
|
76
|
+
...(options.fadeOutSeconds ? { fadeOutSeconds: options.fadeOutSeconds } : {}),
|
|
77
|
+
}));
|
|
78
|
+
|
|
79
|
+
return { id: options.trackId ?? 'captions', kind: 'overlay', order: options.order ?? 100, clips };
|
|
80
|
+
}
|