@open-artifact/motion-audio 0.0.0-stage → 0.12.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +5 -0
- package/README.md +5 -2
- package/package.json +38 -4
- package/src/index.d.ts +46 -0
- package/src/index.js +25 -0
- package/src/keepalive.d.ts +10 -0
- package/src/keepalive.js +49 -0
- package/src/loudness.d.ts +32 -0
- package/src/loudness.js +108 -0
- package/src/midi.d.ts +16 -0
- package/src/midi.js +179 -0
- package/src/mixer.d.ts +29 -0
- package/src/mixer.js +70 -0
- package/src/narration/captions.d.ts +24 -0
- package/src/narration/captions.js +45 -0
- package/src/narration/decode.d.ts +25 -0
- package/src/narration/decode.js +72 -0
- package/src/narration/duck.d.ts +20 -0
- package/src/narration/duck.js +34 -0
- package/src/narration/limits.d.ts +23 -0
- package/src/narration/limits.js +16 -0
- package/src/narration/peaks.d.ts +14 -0
- package/src/narration/peaks.js +27 -0
- package/src/narration/plan.d.ts +12 -0
- package/src/narration/plan.js +31 -0
- package/src/narration/speech.d.ts +31 -0
- package/src/narration/speech.js +24 -0
- package/src/offline.d.ts +79 -0
- package/src/offline.js +257 -0
- package/src/playback-clock.d.ts +39 -0
- package/src/playback-clock.js +90 -0
- package/src/played-score.d.ts +28 -0
- package/src/played-score.js +31 -0
- package/src/player.d.ts +47 -0
- package/src/player.js +237 -0
- package/src/reverb.d.ts +10 -0
- package/src/reverb.js +38 -0
- package/src/scheduler.d.ts +29 -0
- package/src/scheduler.js +64 -0
- package/src/session.d.ts +24 -0
- package/src/session.js +133 -0
- package/src/synth/drums.d.ts +4 -0
- package/src/synth/drums.js +97 -0
- package/src/synth/envelope.d.ts +26 -0
- package/src/synth/envelope.js +39 -0
- package/src/synth/fx.d.ts +9 -0
- package/src/synth/fx.js +131 -0
- package/src/synth/nodes.d.ts +22 -0
- package/src/synth/nodes.js +63 -0
- package/src/synth/noise.d.ts +6 -0
- package/src/synth/noise.js +31 -0
- package/src/synth/patches.d.ts +5 -0
- package/src/synth/patches.js +119 -0
- package/src/synth/presets.d.ts +32 -0
- package/src/synth/presets.js +136 -0
- package/src/synth/schedule.d.ts +5 -0
- package/src/synth/schedule.js +25 -0
- package/src/synth/voice.d.ts +35 -0
- package/src/synth/voice.js +52 -0
- package/src/timeline.d.ts +25 -0
- package/src/timeline.js +49 -0
- package/src/wave.d.ts +17 -0
- package/src/wave.js +111 -0
- package/src/window.d.ts +32 -0
- package/src/window.js +59 -0
package/src/midi.js
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A score as a Standard MIDI File (format 1, 480 ticks a beat): a conductor
|
|
3
|
+
* track with the tempo and where film time 0 falls, a track per patch with
|
|
4
|
+
* its General MIDI program, and the drums and effects on channel 10.
|
|
5
|
+
* Bells play as the notes they ring; a reverse swell lands where it is first
|
|
6
|
+
* heard. The score's tempo sets the beat, so its bars line up in a DAW.
|
|
7
|
+
*/
|
|
8
|
+
import { MAX_EVENT_SECONDS } from "./played-score.js";
|
|
9
|
+
import { soundExtent } from "./synth/presets.js";
|
|
10
|
+
import { voiceOf } from "./synth/voice.js";
|
|
11
|
+
export const MIDI_TICKS_PER_BEAT = 480;
|
|
12
|
+
export const DEFAULT_MIDI_TITLE = 'animation score';
|
|
13
|
+
const DEFAULT_BPM = 120;
|
|
14
|
+
const BPM_RANGE = [4, 960];
|
|
15
|
+
const DRUM_CHANNEL = 9;
|
|
16
|
+
const DRUM_TRACK_NAME = 'drums + foley';
|
|
17
|
+
/** General MIDI programs (0-based) of the patches. */
|
|
18
|
+
const PROGRAMS = { marimba: 12, ep: 4, bass: 33, pad: 89, vibes: 11, glock: 9, pluck: 108, keys: 81 };
|
|
19
|
+
/** General MIDI percussion keys of the drums. */
|
|
20
|
+
const DRUM_KEYS = {
|
|
21
|
+
kick: 36,
|
|
22
|
+
snare: 38,
|
|
23
|
+
clap: 39,
|
|
24
|
+
hat: 42,
|
|
25
|
+
openhat: 46,
|
|
26
|
+
shaker: 82,
|
|
27
|
+
sidestick: 37,
|
|
28
|
+
woodblock: 76,
|
|
29
|
+
tamb: 54,
|
|
30
|
+
tom: 45,
|
|
31
|
+
crash: 49,
|
|
32
|
+
ride: 51,
|
|
33
|
+
cowbell: 56,
|
|
34
|
+
};
|
|
35
|
+
const DRUM_SECONDS = 0.1;
|
|
36
|
+
/** Percussion keys that stand in for the effects. */
|
|
37
|
+
const FX_KEYS = { tick: 77, click: 37, doubleclick: 37, pop: 76, thud: 41, whoosh: 55, revswell: 55, boing: 47, roll: 77 };
|
|
38
|
+
const FX_KEY = 77;
|
|
39
|
+
/** Bells: the intervals they ring over their root, one after another. */
|
|
40
|
+
const BELLS = {
|
|
41
|
+
ding: { intervals: [0, 7], step: 0.12, root: 88 },
|
|
42
|
+
chime: { intervals: [0, 4, 7, 12], step: 0.09, root: 88 },
|
|
43
|
+
sparkle: { intervals: [0, 7, 12, 16, 19, 24], step: 0.055, root: 96 },
|
|
44
|
+
};
|
|
45
|
+
const BELL_SECONDS = 0.3;
|
|
46
|
+
const BELL_PATCH = 'glock';
|
|
47
|
+
const DOUBLE_CLICK_GAP = 0.11;
|
|
48
|
+
const DOUBLE_CLICK_SECONDS = 0.05;
|
|
49
|
+
const clamp = (value, least, most) => Math.max(least, Math.min(most, value));
|
|
50
|
+
function variableLength(value) {
|
|
51
|
+
const bytes = [value & 0x7f];
|
|
52
|
+
for (let rest = value >>> 7; rest > 0; rest >>>= 7)
|
|
53
|
+
bytes.unshift((rest & 0x7f) | 0x80);
|
|
54
|
+
return bytes;
|
|
55
|
+
}
|
|
56
|
+
const text = (value) => Array.from(new TextEncoder().encode(value));
|
|
57
|
+
const meta = (tick, type, data) => ({ tick, order: 0, bytes: [0xff, type, ...variableLength(data.length), ...data] });
|
|
58
|
+
function trackChunk(events) {
|
|
59
|
+
events.sort((first, second) => first.tick - second.tick || first.order - second.order);
|
|
60
|
+
const body = [];
|
|
61
|
+
let last = 0;
|
|
62
|
+
for (const event of events) {
|
|
63
|
+
body.push(...variableLength(event.tick - last), ...event.bytes);
|
|
64
|
+
last = event.tick;
|
|
65
|
+
}
|
|
66
|
+
body.push(0x00, 0xff, 0x2f, 0x00);
|
|
67
|
+
return [...text('MTrk'), ...word(body.length), ...body];
|
|
68
|
+
}
|
|
69
|
+
const word = (value) => [(value >>> 24) & 0xff, (value >>> 16) & 0xff, (value >>> 8) & 0xff, value & 0xff];
|
|
70
|
+
/** Ends a note where the same key starts again on its channel; of two starting together, the louder stays. */
|
|
71
|
+
function untangle(notes) {
|
|
72
|
+
notes.sort((first, second) => first.on - second.on);
|
|
73
|
+
const sounding = new Map();
|
|
74
|
+
const kept = [];
|
|
75
|
+
for (const note of notes) {
|
|
76
|
+
const slot = note.channel * 128 + note.key;
|
|
77
|
+
const before = sounding.get(slot);
|
|
78
|
+
if (before !== undefined && before.on === note.on) {
|
|
79
|
+
before.velocity = Math.max(before.velocity, note.velocity);
|
|
80
|
+
before.off = Math.max(before.off, note.off);
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
if (before !== undefined && before.off > note.on)
|
|
84
|
+
before.off = note.on;
|
|
85
|
+
sounding.set(slot, note);
|
|
86
|
+
kept.push(note);
|
|
87
|
+
}
|
|
88
|
+
return kept;
|
|
89
|
+
}
|
|
90
|
+
/** The score as a Standard MIDI File. */
|
|
91
|
+
export function toMidi(score, options = {}) {
|
|
92
|
+
const bpm = clamp(Number.isFinite(score.tempo?.bpm) ? score.tempo.bpm : DEFAULT_BPM, BPM_RANGE[0], BPM_RANGE[1]);
|
|
93
|
+
const t0 = Number.isFinite(score.tempo?.t0) ? score.tempo.t0 : 0;
|
|
94
|
+
const secondsPerBeat = 60 / bpm;
|
|
95
|
+
const beatsBefore = Math.ceil(Math.max(0, t0) / secondsPerBeat - 1e-9);
|
|
96
|
+
// Film time 0 lands this far into the file, so the score's beat 0 starts a bar.
|
|
97
|
+
const offset = t0 - beatsBefore * secondsPerBeat;
|
|
98
|
+
const tick = (seconds) => Math.max(0, Math.round(((seconds - offset) / secondsPerBeat) * MIDI_TICKS_PER_BEAT));
|
|
99
|
+
const lanes = new Map();
|
|
100
|
+
const drums = { name: DRUM_TRACK_NAME, channel: DRUM_CHANNEL, program: null, notes: [] };
|
|
101
|
+
let nextChannel = 0;
|
|
102
|
+
const lane = (patch) => {
|
|
103
|
+
let found = lanes.get(patch);
|
|
104
|
+
if (found === undefined) {
|
|
105
|
+
found = { name: patch, channel: nextChannel, program: PROGRAMS[patch], notes: [] };
|
|
106
|
+
nextChannel = Math.min(15, nextChannel + 1 === DRUM_CHANNEL ? DRUM_CHANNEL + 1 : nextChannel + 1);
|
|
107
|
+
lanes.set(patch, found);
|
|
108
|
+
}
|
|
109
|
+
return found;
|
|
110
|
+
};
|
|
111
|
+
const play = (into, key, velocity, at, seconds) => {
|
|
112
|
+
const on = tick(at);
|
|
113
|
+
into.notes.push({
|
|
114
|
+
channel: into.channel,
|
|
115
|
+
key: clamp(Math.round(key), 0, 127),
|
|
116
|
+
velocity: clamp(Math.round(velocity), 1, 127),
|
|
117
|
+
on,
|
|
118
|
+
off: Math.max(on + 1, tick(at + Math.max(0.02, seconds)) - 1),
|
|
119
|
+
});
|
|
120
|
+
};
|
|
121
|
+
const events = score.events
|
|
122
|
+
.filter((event) => Number.isFinite(event.t) && Number.isFinite(event.dur) && event.t >= 0 && event.t < MAX_EVENT_SECONDS)
|
|
123
|
+
.sort((first, second) => first.t - second.t);
|
|
124
|
+
for (const event of events) {
|
|
125
|
+
const voice = voiceOf(event);
|
|
126
|
+
if (voice === null)
|
|
127
|
+
continue;
|
|
128
|
+
const { patch, sound } = voice;
|
|
129
|
+
const at = voice.t;
|
|
130
|
+
switch (patch.kind) {
|
|
131
|
+
case 'note':
|
|
132
|
+
play(lane(patch.name), sound.midi, sound.vel, at, sound.dur);
|
|
133
|
+
break;
|
|
134
|
+
case 'drum':
|
|
135
|
+
play(drums, DRUM_KEYS[patch.name], sound.vel, at, DRUM_SECONDS);
|
|
136
|
+
break;
|
|
137
|
+
case 'fx': {
|
|
138
|
+
const bell = BELLS[patch.name];
|
|
139
|
+
if (bell !== undefined) {
|
|
140
|
+
const root = typeof event.midi === 'number' && event.midi > 0 ? sound.midi : bell.root;
|
|
141
|
+
bell.intervals.forEach((interval, index) => play(lane(BELL_PATCH), root + interval, sound.vel, at + index * bell.step, BELL_SECONDS));
|
|
142
|
+
}
|
|
143
|
+
else if (patch.name === 'doubleclick') {
|
|
144
|
+
play(drums, FX_KEYS.doubleclick, sound.vel, at, DOUBLE_CLICK_SECONDS);
|
|
145
|
+
play(drums, FX_KEYS.doubleclick, sound.vel, at + DOUBLE_CLICK_GAP, DOUBLE_CLICK_SECONDS);
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
const [heard] = soundExtent(patch, at, sound.dur);
|
|
149
|
+
play(drums, FX_KEYS[patch.name] ?? FX_KEY, sound.vel, Math.max(0, heard), sound.dur);
|
|
150
|
+
}
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
default: {
|
|
154
|
+
const never = patch;
|
|
155
|
+
return never;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
const microsPerBeat = Math.round(60_000_000 / bpm);
|
|
160
|
+
const conductor = [
|
|
161
|
+
meta(0, 0x03, text(options.title?.trim() || DEFAULT_MIDI_TITLE)),
|
|
162
|
+
meta(0, 0x51, [(microsPerBeat >>> 16) & 0xff, (microsPerBeat >>> 8) & 0xff, microsPerBeat & 0xff]),
|
|
163
|
+
meta(0, 0x58, [4, 2, 24, 8]),
|
|
164
|
+
meta(tick(0), 0x06, text(`film t=0 (${(-offset).toFixed(3)} s into the file)`)),
|
|
165
|
+
];
|
|
166
|
+
const tracks = [trackChunk(conductor)];
|
|
167
|
+
for (const each of [...lanes.values(), ...(drums.notes.length > 0 ? [drums] : [])]) {
|
|
168
|
+
const trackEvents = [meta(0, 0x03, text(each.name))];
|
|
169
|
+
if (each.program !== null)
|
|
170
|
+
trackEvents.push({ tick: 0, order: 0, bytes: [0xc0 | each.channel, each.program] });
|
|
171
|
+
for (const note of untangle(each.notes)) {
|
|
172
|
+
trackEvents.push({ tick: note.on, order: 2, bytes: [0x90 | note.channel, note.key, note.velocity] });
|
|
173
|
+
trackEvents.push({ tick: note.off, order: 1, bytes: [0x80 | note.channel, note.key, 0] });
|
|
174
|
+
}
|
|
175
|
+
tracks.push(trackChunk(trackEvents));
|
|
176
|
+
}
|
|
177
|
+
const header = [...text('MThd'), ...word(6), 0, 1, (tracks.length >>> 8) & 0xff, tracks.length & 0xff, MIDI_TICKS_PER_BEAT >>> 8, MIDI_TICKS_PER_BEAT & 0xff];
|
|
178
|
+
return new Uint8Array([...header, ...tracks.flat()]);
|
|
179
|
+
}
|
package/src/mixer.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The master bus. The score's dry and reverb sends meet in a gentle compressor, take the loudness normalization gain, are
|
|
3
|
+
* ducked under narration, then join the narration in a limiter, and a soft clipper keeps the sum under full scale.
|
|
4
|
+
*/
|
|
5
|
+
export interface MasterBus {
|
|
6
|
+
readonly context: BaseAudioContext;
|
|
7
|
+
/** The score's dry send. */
|
|
8
|
+
readonly input: GainNode;
|
|
9
|
+
/** The score's reverb send. */
|
|
10
|
+
readonly reverbInput: GainNode;
|
|
11
|
+
/** The score's loudness normalization gain. */
|
|
12
|
+
readonly normalization: GainNode;
|
|
13
|
+
/** The score's level under narration. */
|
|
14
|
+
readonly duck: GainNode;
|
|
15
|
+
/** Narration's input, past the score's compressor, normalization and ducking. */
|
|
16
|
+
readonly speech: GainNode;
|
|
17
|
+
/** The listener's volume (0 when muted). */
|
|
18
|
+
readonly output: GainNode;
|
|
19
|
+
}
|
|
20
|
+
/** Linear up to 0.85, then bending smoothly towards 1 (tanh), over inputs −2…2. */
|
|
21
|
+
export declare function softClipCurve(): Float32Array<ArrayBuffer>;
|
|
22
|
+
/** The score's master level in dB, kept within −40…6, as a gain. */
|
|
23
|
+
export declare function masterGain(decibels: number): number;
|
|
24
|
+
/**
|
|
25
|
+
* The score's level at play time `play` given its master level and a fade-out over the last `fadeOut` seconds of
|
|
26
|
+
* `duration`: the level to resume from when playback starts inside the fade.
|
|
27
|
+
*/
|
|
28
|
+
export declare function fadeLevel(level: number, fadeOut: number, duration: number, play: number): number;
|
|
29
|
+
export declare function createMasterBus(context: BaseAudioContext, destination?: AudioNode): MasterBus;
|
package/src/mixer.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { createReverb } from "./reverb.js";
|
|
2
|
+
const CLIP_KNEE = 0.85;
|
|
3
|
+
const CLIP_POINTS = 2049;
|
|
4
|
+
/** The limiter's output is halved before the clipper, whose curve spans −2…2. */
|
|
5
|
+
const CLIP_INPUT_GAIN = 0.5;
|
|
6
|
+
/** Linear up to 0.85, then bending smoothly towards 1 (tanh), over inputs −2…2. */
|
|
7
|
+
export function softClipCurve() {
|
|
8
|
+
const curve = new Float32Array(CLIP_POINTS);
|
|
9
|
+
for (let index = 0; index < CLIP_POINTS; index++) {
|
|
10
|
+
const input = (index / (CLIP_POINTS - 1)) * 4 - 2;
|
|
11
|
+
const size = Math.abs(input);
|
|
12
|
+
curve[index] = Math.sign(input) * (size < CLIP_KNEE ? size : CLIP_KNEE + (1 - CLIP_KNEE) * Math.tanh((size - CLIP_KNEE) / (1 - CLIP_KNEE)));
|
|
13
|
+
}
|
|
14
|
+
return curve;
|
|
15
|
+
}
|
|
16
|
+
/** The score's master level in dB, kept within −40…6, as a gain. */
|
|
17
|
+
export function masterGain(decibels) {
|
|
18
|
+
return Math.pow(10, Math.max(-40, Math.min(6, decibels)) / 20);
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* The score's level at play time `play` given its master level and a fade-out over the last `fadeOut` seconds of
|
|
22
|
+
* `duration`: the level to resume from when playback starts inside the fade.
|
|
23
|
+
*/
|
|
24
|
+
export function fadeLevel(level, fadeOut, duration, play) {
|
|
25
|
+
const fade = Math.max(0, Math.min(duration, fadeOut));
|
|
26
|
+
if (duration <= 0 || fade <= 0)
|
|
27
|
+
return level;
|
|
28
|
+
const fadeStart = duration - fade;
|
|
29
|
+
if (play <= fadeStart)
|
|
30
|
+
return level;
|
|
31
|
+
if (play >= duration)
|
|
32
|
+
return 0;
|
|
33
|
+
return level * (1 - (play - fadeStart) / fade);
|
|
34
|
+
}
|
|
35
|
+
export function createMasterBus(context, destination = context.destination) {
|
|
36
|
+
const input = context.createGain();
|
|
37
|
+
const reverb = createReverb(context);
|
|
38
|
+
const compressor = context.createDynamicsCompressor();
|
|
39
|
+
compressor.threshold.value = -16;
|
|
40
|
+
compressor.knee.value = 10;
|
|
41
|
+
compressor.ratio.value = 2.5;
|
|
42
|
+
compressor.attack.value = 0.008;
|
|
43
|
+
compressor.release.value = 0.2;
|
|
44
|
+
input.connect(compressor);
|
|
45
|
+
reverb.output.connect(compressor);
|
|
46
|
+
const normalization = context.createGain();
|
|
47
|
+
const duck = context.createGain();
|
|
48
|
+
const speech = context.createGain();
|
|
49
|
+
const limiter = context.createDynamicsCompressor();
|
|
50
|
+
limiter.threshold.value = -4;
|
|
51
|
+
limiter.knee.value = 2;
|
|
52
|
+
limiter.ratio.value = 20;
|
|
53
|
+
limiter.attack.value = 0.002;
|
|
54
|
+
limiter.release.value = 0.08;
|
|
55
|
+
const clipInput = context.createGain();
|
|
56
|
+
clipInput.gain.value = CLIP_INPUT_GAIN;
|
|
57
|
+
const clipper = context.createWaveShaper();
|
|
58
|
+
clipper.curve = softClipCurve();
|
|
59
|
+
clipper.oversample = 'none';
|
|
60
|
+
const output = context.createGain();
|
|
61
|
+
compressor.connect(normalization);
|
|
62
|
+
normalization.connect(duck);
|
|
63
|
+
duck.connect(limiter);
|
|
64
|
+
speech.connect(limiter);
|
|
65
|
+
limiter.connect(clipInput);
|
|
66
|
+
clipInput.connect(clipper);
|
|
67
|
+
clipper.connect(output);
|
|
68
|
+
output.connect(destination);
|
|
69
|
+
return { context, input, reverbInput: reverb.input, normalization, duck, speech, output };
|
|
70
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { PlaySeconds } from '@open-artifact/motion-core';
|
|
2
|
+
import type { FilmTimeline } from '../timeline.ts';
|
|
3
|
+
import type { NarrationPlan } from './plan.ts';
|
|
4
|
+
import type { DecodedNarration } from './speech.ts';
|
|
5
|
+
/** A caption on screen from `start` to `end`, in play time. */
|
|
6
|
+
export interface CaptionCue {
|
|
7
|
+
readonly start: PlaySeconds;
|
|
8
|
+
readonly end: PlaySeconds;
|
|
9
|
+
readonly text: string;
|
|
10
|
+
}
|
|
11
|
+
/** A line on the voiceover lane: when it plays, how long its take runs, its words and its waveform (null: not decoded). */
|
|
12
|
+
export interface VoiceoverLine {
|
|
13
|
+
readonly at: PlaySeconds;
|
|
14
|
+
readonly dur: number;
|
|
15
|
+
readonly text: string;
|
|
16
|
+
readonly audio: string;
|
|
17
|
+
readonly peaks: readonly number[] | null;
|
|
18
|
+
}
|
|
19
|
+
/** The captions of a narration that shows them: each line with words, from its start for its said length. */
|
|
20
|
+
export declare function captionCues(plan: NarrationPlan | null, timeline: FilmTimeline): CaptionCue[];
|
|
21
|
+
/** The caption showing at `t`: the latest one started by then and not yet over. */
|
|
22
|
+
export declare function cueAt(cues: readonly CaptionCue[], t: PlaySeconds): CaptionCue | null;
|
|
23
|
+
/** The voiceover lane's lines, with each take's real length where it decoded. */
|
|
24
|
+
export declare function voiceoverLines(narration: DecodedNarration, timeline: FilmTimeline): VoiceoverLine[];
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { playSeconds } from '@open-artifact/motion-core';
|
|
2
|
+
/** The captions of a narration that shows them: each line with words, from its start for its said length. */
|
|
3
|
+
export function captionCues(plan, timeline) {
|
|
4
|
+
if (plan === null || plan.captions === false)
|
|
5
|
+
return [];
|
|
6
|
+
const cues = [];
|
|
7
|
+
for (const line of plan.lines) {
|
|
8
|
+
if (line.text.trim() === '')
|
|
9
|
+
continue;
|
|
10
|
+
const start = timeline.toPlay(line.at);
|
|
11
|
+
if (start === null)
|
|
12
|
+
continue;
|
|
13
|
+
cues.push({ start, end: playSeconds(start + line.dur), text: line.text });
|
|
14
|
+
}
|
|
15
|
+
return cues;
|
|
16
|
+
}
|
|
17
|
+
/** The caption showing at `t`: the latest one started by then and not yet over. */
|
|
18
|
+
export function cueAt(cues, t) {
|
|
19
|
+
let showing = null;
|
|
20
|
+
for (const cue of cues) {
|
|
21
|
+
if (cue.start > t)
|
|
22
|
+
break;
|
|
23
|
+
if (t < cue.end)
|
|
24
|
+
showing = cue;
|
|
25
|
+
}
|
|
26
|
+
return showing;
|
|
27
|
+
}
|
|
28
|
+
/** The voiceover lane's lines, with each take's real length where it decoded. */
|
|
29
|
+
export function voiceoverLines(narration, timeline) {
|
|
30
|
+
const lines = [];
|
|
31
|
+
for (const line of narration.plan.lines) {
|
|
32
|
+
const at = timeline.toPlay(line.at);
|
|
33
|
+
if (at === null)
|
|
34
|
+
continue;
|
|
35
|
+
const take = narration.takes.get(line.audio);
|
|
36
|
+
lines.push({
|
|
37
|
+
at,
|
|
38
|
+
dur: take?.duration ?? line.dur,
|
|
39
|
+
text: line.text,
|
|
40
|
+
audio: line.audio,
|
|
41
|
+
peaks: narration.peaks.get(line.audio) ?? null,
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
return lines;
|
|
45
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { PcmWave } from '../wave.ts';
|
|
2
|
+
import type { NarrationLimits } from './limits.ts';
|
|
3
|
+
import type { NarrationPlan } from './plan.ts';
|
|
4
|
+
import type { DecodedNarration, TakeIssue } from './speech.ts';
|
|
5
|
+
/** What turns a take's bytes into samples: any audio context. */
|
|
6
|
+
export interface TakeDecoder {
|
|
7
|
+
decodeAudioData(bytes: ArrayBuffer): Promise<AudioBuffer>;
|
|
8
|
+
}
|
|
9
|
+
export interface DecodeOptions {
|
|
10
|
+
readonly limits?: NarrationLimits;
|
|
11
|
+
/** False once the film has moved on, so a stale decode stops early. */
|
|
12
|
+
readonly isCurrent?: () => boolean;
|
|
13
|
+
readonly decoder?: TakeDecoder | null;
|
|
14
|
+
/** How the takes' loudness gain is found; by default they are rendered through the mix and measured. */
|
|
15
|
+
readonly speechGain?: (takes: readonly AudioBuffer[], targetLufs: number) => Promise<number>;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Judges a take by its bytes before decoding: too big unless it is a plain wave within the wave limit, too long when a
|
|
19
|
+
* wave says so, or past the narration's total.
|
|
20
|
+
*/
|
|
21
|
+
export declare function judgeTakeBytes(byteLength: number, wave: PcmWave | null, totalSeconds: number, limits: NarrationLimits): TakeIssue | null;
|
|
22
|
+
/** Judges a decoded take by its length. */
|
|
23
|
+
export declare function judgeTakeSeconds(seconds: number | null, totalSeconds: number, limits: NarrationLimits): TakeIssue | null;
|
|
24
|
+
/** Decodes each take the lines name once, within the limits. Null when the film moved on meanwhile. */
|
|
25
|
+
export declare function decodeNarration(plan: NarrationPlan, assets: ReadonlyMap<string, ArrayBuffer>, options?: DecodeOptions): Promise<DecodedNarration | null>;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { measureSpeechGain } from "../offline.js";
|
|
2
|
+
import { parsePcmWave } from "../wave.js";
|
|
3
|
+
import { DEFAULT_NARRATION_LIMITS } from "./limits.js";
|
|
4
|
+
import { voicePeaks } from "./peaks.js";
|
|
5
|
+
const KIB = 1024;
|
|
6
|
+
/** A take may run this much past a line's limit before it is too long. */
|
|
7
|
+
const LINE_SLACK_SECONDS = 1;
|
|
8
|
+
/**
|
|
9
|
+
* Judges a take by its bytes before decoding: too big unless it is a plain wave within the wave limit, too long when a
|
|
10
|
+
* wave says so, or past the narration's total.
|
|
11
|
+
*/
|
|
12
|
+
export function judgeTakeBytes(byteLength, wave, totalSeconds, limits) {
|
|
13
|
+
if (wave === null && byteLength > limits.maxTakeKib * KIB)
|
|
14
|
+
return 'big';
|
|
15
|
+
if (wave !== null && wave.seconds > limits.maxLineSeconds + LINE_SLACK_SECONDS)
|
|
16
|
+
return 'long';
|
|
17
|
+
if (totalSeconds + (wave?.seconds ?? 0) > limits.maxSpeechSeconds || totalSeconds >= limits.maxSpeechSeconds)
|
|
18
|
+
return 'total';
|
|
19
|
+
return null;
|
|
20
|
+
}
|
|
21
|
+
/** Judges a decoded take by its length. */
|
|
22
|
+
export function judgeTakeSeconds(seconds, totalSeconds, limits) {
|
|
23
|
+
if (seconds === null || seconds === 0)
|
|
24
|
+
return 'undecodable';
|
|
25
|
+
if (seconds > limits.maxLineSeconds + LINE_SLACK_SECONDS)
|
|
26
|
+
return 'long';
|
|
27
|
+
if (totalSeconds + seconds > limits.maxSpeechSeconds)
|
|
28
|
+
return 'total';
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
function defaultDecoder() {
|
|
32
|
+
return typeof OfflineAudioContext === 'undefined' ? null : new OfflineAudioContext(1, 1, 48000);
|
|
33
|
+
}
|
|
34
|
+
/** Decodes each take the lines name once, within the limits. Null when the film moved on meanwhile. */
|
|
35
|
+
export async function decodeNarration(plan, assets, options = {}) {
|
|
36
|
+
const limits = options.limits ?? DEFAULT_NARRATION_LIMITS;
|
|
37
|
+
const isCurrent = options.isCurrent ?? (() => true);
|
|
38
|
+
const decoder = options.decoder === undefined ? defaultDecoder() : options.decoder;
|
|
39
|
+
const takes = new Map();
|
|
40
|
+
const found = new Map();
|
|
41
|
+
const peaks = new Map();
|
|
42
|
+
let totalSeconds = 0;
|
|
43
|
+
for (const name of new Set(plan.lines.map((line) => line.audio))) {
|
|
44
|
+
const bytes = assets.get(name);
|
|
45
|
+
if (bytes === undefined)
|
|
46
|
+
continue;
|
|
47
|
+
if (!isCurrent())
|
|
48
|
+
return null;
|
|
49
|
+
// A plain 16-bit wave is read by its chunks, so its length is known before decoding and it may be larger than an mp3.
|
|
50
|
+
const wave = bytes.byteLength <= Math.max(limits.maxTakeKib, limits.maxWavKib) * KIB ? parsePcmWave(bytes) : null;
|
|
51
|
+
const before = judgeTakeBytes(bytes.byteLength, wave, totalSeconds, limits);
|
|
52
|
+
const buffer = before !== null || decoder === null
|
|
53
|
+
? null
|
|
54
|
+
: await decoder.decodeAudioData(wave?.bare() ?? bytes.slice(0)).catch(() => null);
|
|
55
|
+
const issue = before ?? judgeTakeSeconds(buffer === null || buffer.length === 0 ? null : buffer.duration, totalSeconds, limits);
|
|
56
|
+
if (issue !== null || buffer === null) {
|
|
57
|
+
found.set(name, issue ?? 'undecodable');
|
|
58
|
+
continue;
|
|
59
|
+
}
|
|
60
|
+
found.set(name, buffer.duration);
|
|
61
|
+
totalSeconds += buffer.duration;
|
|
62
|
+
takes.set(name, buffer);
|
|
63
|
+
peaks.set(name, voicePeaks(buffer.getChannelData(0), buffer.duration));
|
|
64
|
+
}
|
|
65
|
+
if (!isCurrent())
|
|
66
|
+
return null;
|
|
67
|
+
const measure = options.speechGain ?? ((buffers, target) => measureSpeechGain(buffers, target));
|
|
68
|
+
const gain = takes.size === 0 ? 1 : await measure([...takes.values()], limits.loudness);
|
|
69
|
+
if (!isCurrent())
|
|
70
|
+
return null;
|
|
71
|
+
return { plan, takes, found, peaks, gain };
|
|
72
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { PlaySeconds } from '@open-artifact/motion-core';
|
|
2
|
+
/** A point of the score's level under narration: gain `v` at play time `t`, straight lines between points. */
|
|
3
|
+
export interface DuckPoint {
|
|
4
|
+
readonly t: PlaySeconds;
|
|
5
|
+
readonly v: number;
|
|
6
|
+
}
|
|
7
|
+
/** The score starts going down this long before a line. */
|
|
8
|
+
export declare const DUCK_ATTACK_SECONDS = 0.25;
|
|
9
|
+
/** And comes back up over this long after it. */
|
|
10
|
+
export declare const DUCK_RELEASE_SECONDS = 0.5;
|
|
11
|
+
/**
|
|
12
|
+
* The score's level under narration: down to `duckDb` over the attack before each line, held while it speaks, back up
|
|
13
|
+
* over the release after. Lines close enough that one's attack meets the last one's release stay ducked between them.
|
|
14
|
+
*/
|
|
15
|
+
export declare function duckEnvelope(lines: ReadonlyArray<{
|
|
16
|
+
readonly at: PlaySeconds;
|
|
17
|
+
readonly dur: number;
|
|
18
|
+
}>, duckDb: number): DuckPoint[];
|
|
19
|
+
/** The envelope's level at `time`: before its first point that point's level, after its last the last one's. */
|
|
20
|
+
export declare function sampleEnvelope(points: readonly DuckPoint[], time: number): number;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { playSeconds } from '@open-artifact/motion-core';
|
|
2
|
+
/** The score starts going down this long before a line. */
|
|
3
|
+
export const DUCK_ATTACK_SECONDS = 0.25;
|
|
4
|
+
/** And comes back up over this long after it. */
|
|
5
|
+
export const DUCK_RELEASE_SECONDS = 0.5;
|
|
6
|
+
/**
|
|
7
|
+
* The score's level under narration: down to `duckDb` over the attack before each line, held while it speaks, back up
|
|
8
|
+
* over the release after. Lines close enough that one's attack meets the last one's release stay ducked between them.
|
|
9
|
+
*/
|
|
10
|
+
export function duckEnvelope(lines, duckDb) {
|
|
11
|
+
const low = Math.pow(10, duckDb / 20);
|
|
12
|
+
const points = [];
|
|
13
|
+
let end = -Infinity;
|
|
14
|
+
for (const line of [...lines].sort((first, second) => first.at - second.at)) {
|
|
15
|
+
if (line.at - DUCK_ATTACK_SECONDS <= end + DUCK_RELEASE_SECONDS)
|
|
16
|
+
points.length -= 2;
|
|
17
|
+
else
|
|
18
|
+
points.push({ t: playSeconds(line.at - DUCK_ATTACK_SECONDS), v: 1 }, { t: line.at, v: low });
|
|
19
|
+
end = Math.max(end, line.at + line.dur);
|
|
20
|
+
points.push({ t: playSeconds(end), v: low }, { t: playSeconds(end + DUCK_RELEASE_SECONDS), v: 1 });
|
|
21
|
+
}
|
|
22
|
+
return points;
|
|
23
|
+
}
|
|
24
|
+
/** The envelope's level at `time`: before its first point that point's level, after its last the last one's. */
|
|
25
|
+
export function sampleEnvelope(points, time) {
|
|
26
|
+
let previous = null;
|
|
27
|
+
for (const point of points) {
|
|
28
|
+
if (time < point.t) {
|
|
29
|
+
return previous === null ? point.v : previous.v + ((point.v - previous.v) * (time - previous.t)) / (point.t - previous.t);
|
|
30
|
+
}
|
|
31
|
+
previous = point;
|
|
32
|
+
}
|
|
33
|
+
return previous?.v ?? 1;
|
|
34
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/** What narration may hold, and how it is mixed. */
|
|
2
|
+
export interface NarrationLimits {
|
|
3
|
+
/** How far the score is lowered under narration, in dB, when the narration names none. */
|
|
4
|
+
readonly duckDb: number;
|
|
5
|
+
/** The narration's loudness target, in LUFS. */
|
|
6
|
+
readonly loudness: number;
|
|
7
|
+
readonly maxLines: number;
|
|
8
|
+
readonly maxLineSeconds: number;
|
|
9
|
+
/** All takes together. */
|
|
10
|
+
readonly maxSpeechSeconds: number;
|
|
11
|
+
readonly maxLineChars: number;
|
|
12
|
+
/** A take's size in KiB, unless it is a plain 16-bit PCM wave. */
|
|
13
|
+
readonly maxTakeKib: number;
|
|
14
|
+
/** A plain 16-bit PCM wave's size in KiB. */
|
|
15
|
+
readonly maxWavKib: number;
|
|
16
|
+
}
|
|
17
|
+
export declare const DEFAULT_NARRATION_LIMITS: NarrationLimits;
|
|
18
|
+
/** The duck a narration may ask for, in dB. */
|
|
19
|
+
export declare const DUCK_BOUNDS: readonly [number, number];
|
|
20
|
+
/** A line's words as shown: this many UTF-16 units at most. */
|
|
21
|
+
export declare const MAX_CAPTION_UNITS = 600;
|
|
22
|
+
/** A take's asset name is cut to this many characters. */
|
|
23
|
+
export declare const MAX_TAKE_NAME = 64;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export const DEFAULT_NARRATION_LIMITS = {
|
|
2
|
+
duckDb: -12,
|
|
3
|
+
loudness: -16,
|
|
4
|
+
maxLines: 100,
|
|
5
|
+
maxLineSeconds: 30,
|
|
6
|
+
maxSpeechSeconds: 300,
|
|
7
|
+
maxLineChars: 300,
|
|
8
|
+
maxTakeKib: 256,
|
|
9
|
+
maxWavKib: 1536,
|
|
10
|
+
};
|
|
11
|
+
/** The duck a narration may ask for, in dB. */
|
|
12
|
+
export const DUCK_BOUNDS = [-30, 0];
|
|
13
|
+
/** A line's words as shown: this many UTF-16 units at most. */
|
|
14
|
+
export const MAX_CAPTION_UNITS = 600;
|
|
15
|
+
/** A take's asset name is cut to this many characters. */
|
|
16
|
+
export const MAX_TAKE_NAME = 64;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export declare const PEAKS_PER_SECOND = 60;
|
|
2
|
+
export declare const LEAST_PEAKS = 8;
|
|
3
|
+
export declare const MOST_PEAKS = 1800;
|
|
4
|
+
/** No bar is drawn lower than this, so silence still shows the take. */
|
|
5
|
+
export declare const PEAK_FLOOR = 0.08;
|
|
6
|
+
/** Lifts quiet speech so it still shows. */
|
|
7
|
+
export declare const PEAK_GAMMA = 0.6;
|
|
8
|
+
/** Samples read per bar at most. */
|
|
9
|
+
export declare const SAMPLES_PER_PEAK = 256;
|
|
10
|
+
/**
|
|
11
|
+
* A take's waveform for the voiceover lane: bars at 60 a second (8 to 1800), each the loudest of the samples read across
|
|
12
|
+
* its span, scaled to the take's loudest bar and lifted by the gamma.
|
|
13
|
+
*/
|
|
14
|
+
export declare function voicePeaks(samples: Float32Array, seconds: number): number[];
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
export const PEAKS_PER_SECOND = 60;
|
|
2
|
+
export const LEAST_PEAKS = 8;
|
|
3
|
+
export const MOST_PEAKS = 1800;
|
|
4
|
+
/** No bar is drawn lower than this, so silence still shows the take. */
|
|
5
|
+
export const PEAK_FLOOR = 0.08;
|
|
6
|
+
/** Lifts quiet speech so it still shows. */
|
|
7
|
+
export const PEAK_GAMMA = 0.6;
|
|
8
|
+
/** Samples read per bar at most. */
|
|
9
|
+
export const SAMPLES_PER_PEAK = 256;
|
|
10
|
+
/**
|
|
11
|
+
* A take's waveform for the voiceover lane: bars at 60 a second (8 to 1800), each the loudest of the samples read across
|
|
12
|
+
* its span, scaled to the take's loudest bar and lifted by the gamma.
|
|
13
|
+
*/
|
|
14
|
+
export function voicePeaks(samples, seconds) {
|
|
15
|
+
const count = Math.min(MOST_PEAKS, Math.max(LEAST_PEAKS, Math.round(seconds * PEAKS_PER_SECOND)));
|
|
16
|
+
const span = samples.length / count;
|
|
17
|
+
const stride = Math.max(1, Math.floor(span / SAMPLES_PER_PEAK));
|
|
18
|
+
const bars = Array.from({ length: count }, (_unused, index) => {
|
|
19
|
+
let peak = 0;
|
|
20
|
+
for (let sample = Math.floor(index * span), end = Math.floor((index + 1) * span); sample < end; sample += stride) {
|
|
21
|
+
peak = Math.max(peak, Math.abs(samples[sample] ?? 0));
|
|
22
|
+
}
|
|
23
|
+
return peak;
|
|
24
|
+
});
|
|
25
|
+
const loudest = Math.max(...bars);
|
|
26
|
+
return bars.map((bar) => (loudest === 0 ? PEAK_FLOOR : Math.max(PEAK_FLOOR, Number(((bar / loudest) ** PEAK_GAMMA).toFixed(3)))));
|
|
27
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { FilmSeconds, NarrationData, NarrationLine } from '@open-artifact/motion-core';
|
|
2
|
+
import type { NarrationLimits } from './limits.ts';
|
|
3
|
+
/** Narration as it is played: its lines within the film and the limits, in time order, and the duck in dB. */
|
|
4
|
+
export interface NarrationPlan {
|
|
5
|
+
readonly lines: readonly NarrationLine[];
|
|
6
|
+
readonly duckDb: number;
|
|
7
|
+
readonly captions: NarrationData['captions'];
|
|
8
|
+
}
|
|
9
|
+
/** A line's words, cut to the caption limit without splitting a surrogate pair. */
|
|
10
|
+
export declare function clipCaption(text: string): string;
|
|
11
|
+
/** Keeps the lines that start within the film and are no longer than a line may be; null when none is left. */
|
|
12
|
+
export declare function planNarration(narration: NarrationData, duration: FilmSeconds, limits?: NarrationLimits): NarrationPlan | null;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { DEFAULT_NARRATION_LIMITS, DUCK_BOUNDS, MAX_CAPTION_UNITS, MAX_TAKE_NAME } from "./limits.js";
|
|
2
|
+
/** A line's words, cut to the caption limit without splitting a surrogate pair. */
|
|
3
|
+
export function clipCaption(text) {
|
|
4
|
+
const cut = text.length > MAX_CAPTION_UNITS && /[\ud800-\udbff]/.test(text[MAX_CAPTION_UNITS - 1] ?? '') ? MAX_CAPTION_UNITS - 1 : MAX_CAPTION_UNITS;
|
|
5
|
+
return text.slice(0, cut);
|
|
6
|
+
}
|
|
7
|
+
/** Keeps the lines that start within the film and are no longer than a line may be; null when none is left. */
|
|
8
|
+
export function planNarration(narration, duration, limits = DEFAULT_NARRATION_LIMITS) {
|
|
9
|
+
const lines = [];
|
|
10
|
+
for (const line of narration.lines.slice(0, limits.maxLines)) {
|
|
11
|
+
if (!(line.at >= 0 && line.at < duration))
|
|
12
|
+
continue;
|
|
13
|
+
if (!(line.dur > 0 && line.dur <= limits.maxLineSeconds))
|
|
14
|
+
continue;
|
|
15
|
+
lines.push({
|
|
16
|
+
at: line.at,
|
|
17
|
+
dur: line.dur,
|
|
18
|
+
audio: typeof line.audio === 'string' ? line.audio.slice(0, MAX_TAKE_NAME) : '',
|
|
19
|
+
text: typeof line.text === 'string' ? clipCaption(line.text) : '',
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
if (lines.length === 0)
|
|
23
|
+
return null;
|
|
24
|
+
lines.sort((first, second) => first.at - second.at);
|
|
25
|
+
const duck = narration.duck;
|
|
26
|
+
return {
|
|
27
|
+
lines,
|
|
28
|
+
duckDb: typeof duck === 'number' && Number.isFinite(duck) ? Math.max(DUCK_BOUNDS[0], Math.min(DUCK_BOUNDS[1], duck)) : limits.duckDb,
|
|
29
|
+
captions: narration.captions,
|
|
30
|
+
};
|
|
31
|
+
}
|