@open-artifact/motion-audio 0.0.0-stage → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +5 -0
- package/README.md +5 -2
- package/package.json +36 -4
- package/src/index.d.ts +46 -0
- package/src/index.js +25 -0
- package/src/keepalive.d.ts +10 -0
- package/src/keepalive.js +49 -0
- package/src/loudness.d.ts +32 -0
- package/src/loudness.js +108 -0
- package/src/midi.d.ts +16 -0
- package/src/midi.js +179 -0
- package/src/mixer.d.ts +29 -0
- package/src/mixer.js +70 -0
- package/src/narration/captions.d.ts +24 -0
- package/src/narration/captions.js +45 -0
- package/src/narration/decode.d.ts +25 -0
- package/src/narration/decode.js +72 -0
- package/src/narration/duck.d.ts +20 -0
- package/src/narration/duck.js +34 -0
- package/src/narration/limits.d.ts +23 -0
- package/src/narration/limits.js +16 -0
- package/src/narration/peaks.d.ts +14 -0
- package/src/narration/peaks.js +27 -0
- package/src/narration/plan.d.ts +12 -0
- package/src/narration/plan.js +31 -0
- package/src/narration/speech.d.ts +31 -0
- package/src/narration/speech.js +24 -0
- package/src/offline.d.ts +79 -0
- package/src/offline.js +257 -0
- package/src/playback-clock.d.ts +39 -0
- package/src/playback-clock.js +90 -0
- package/src/played-score.d.ts +28 -0
- package/src/played-score.js +31 -0
- package/src/player.d.ts +47 -0
- package/src/player.js +237 -0
- package/src/reverb.d.ts +10 -0
- package/src/reverb.js +38 -0
- package/src/scheduler.d.ts +29 -0
- package/src/scheduler.js +64 -0
- package/src/session.d.ts +24 -0
- package/src/session.js +133 -0
- package/src/synth/drums.d.ts +4 -0
- package/src/synth/drums.js +97 -0
- package/src/synth/envelope.d.ts +26 -0
- package/src/synth/envelope.js +39 -0
- package/src/synth/fx.d.ts +9 -0
- package/src/synth/fx.js +131 -0
- package/src/synth/nodes.d.ts +22 -0
- package/src/synth/nodes.js +63 -0
- package/src/synth/noise.d.ts +6 -0
- package/src/synth/noise.js +31 -0
- package/src/synth/patches.d.ts +5 -0
- package/src/synth/patches.js +119 -0
- package/src/synth/presets.d.ts +32 -0
- package/src/synth/presets.js +136 -0
- package/src/synth/schedule.d.ts +5 -0
- package/src/synth/schedule.js +25 -0
- package/src/synth/voice.d.ts +35 -0
- package/src/synth/voice.js +52 -0
- package/src/timeline.d.ts +25 -0
- package/src/timeline.js +49 -0
- package/src/wave.d.ts +17 -0
- package/src/wave.js +111 -0
- package/src/window.d.ts +32 -0
- package/src/window.js +59 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { PlaySeconds } from '@open-artifact/motion-core';
|
|
2
|
+
import type { FilmTimeline } from '../timeline.ts';
|
|
3
|
+
import type { DuckPoint } from './duck.ts';
|
|
4
|
+
import type { NarrationPlan } from './plan.ts';
|
|
5
|
+
/** Why a take is silent: over the size limit, past the total, not decodable, or longer than a line may be. */
|
|
6
|
+
export type TakeIssue = 'big' | 'total' | 'undecodable' | 'long';
|
|
7
|
+
/** Narration with its takes decoded. `found` has each take's seconds, or why it is silent; a missing take is not in it. */
|
|
8
|
+
export interface DecodedNarration {
|
|
9
|
+
readonly plan: NarrationPlan;
|
|
10
|
+
readonly takes: ReadonlyMap<string, AudioBuffer>;
|
|
11
|
+
readonly found: ReadonlyMap<string, TakeIssue | number>;
|
|
12
|
+
/** Each decoded take's waveform for the voiceover lane. */
|
|
13
|
+
readonly peaks: ReadonlyMap<string, readonly number[]>;
|
|
14
|
+
/** The gain that brings the takes to the narration's loudness target. */
|
|
15
|
+
readonly gain: number;
|
|
16
|
+
}
|
|
17
|
+
export interface SpeechClip {
|
|
18
|
+
readonly at: PlaySeconds;
|
|
19
|
+
readonly buffer: AudioBuffer;
|
|
20
|
+
}
|
|
21
|
+
/** Narration as the mixer plays it: its takes at their play times, their gain, and the score's duck under them. */
|
|
22
|
+
export interface Speech {
|
|
23
|
+
readonly clips: readonly SpeechClip[];
|
|
24
|
+
readonly gain: number;
|
|
25
|
+
readonly duck: readonly DuckPoint[];
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Places the decoded takes in play time. A line starts on the action it speaks of, so its start moves with the picture;
|
|
29
|
+
* its take keeps its own length. A line whose start is cut out is not spoken. Null when nothing is spoken.
|
|
30
|
+
*/
|
|
31
|
+
export declare function placeSpeech(narration: DecodedNarration | null, timeline: FilmTimeline): Speech | null;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { duckEnvelope } from "./duck.js";
|
|
2
|
+
/**
|
|
3
|
+
* Places the decoded takes in play time. A line starts on the action it speaks of, so its start moves with the picture;
|
|
4
|
+
* its take keeps its own length. A line whose start is cut out is not spoken. Null when nothing is spoken.
|
|
5
|
+
*/
|
|
6
|
+
export function placeSpeech(narration, timeline) {
|
|
7
|
+
if (narration === null)
|
|
8
|
+
return null;
|
|
9
|
+
const clips = [];
|
|
10
|
+
for (const line of narration.plan.lines) {
|
|
11
|
+
const buffer = narration.takes.get(line.audio);
|
|
12
|
+
const at = timeline.toPlay(line.at);
|
|
13
|
+
if (buffer === undefined || at === null)
|
|
14
|
+
continue;
|
|
15
|
+
clips.push({ at, buffer });
|
|
16
|
+
}
|
|
17
|
+
if (clips.length === 0)
|
|
18
|
+
return null;
|
|
19
|
+
return {
|
|
20
|
+
clips,
|
|
21
|
+
gain: narration.gain,
|
|
22
|
+
duck: duckEnvelope(clips.map((clip) => ({ at: clip.at, dur: clip.buffer.duration })), narration.plan.duckDb),
|
|
23
|
+
};
|
|
24
|
+
}
|
package/src/offline.d.ts
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import type { PlaySeconds, ScoreData } from '@open-artifact/motion-core';
|
|
2
|
+
import type { LoudnessMeasurement } from './loudness.ts';
|
|
3
|
+
import type { DecodedNarration, Speech, SpeechClip } from './narration/speech.ts';
|
|
4
|
+
import type { PlayedScore } from './played-score.ts';
|
|
5
|
+
import type { VoiceBus } from './synth/nodes.ts';
|
|
6
|
+
import type { FilmTimeline } from './timeline.ts';
|
|
7
|
+
import type { HeardSpan } from './window.ts';
|
|
8
|
+
/** Everything a film sounds: its score and its narration, both in play time. */
|
|
9
|
+
export interface FilmSound {
|
|
10
|
+
readonly score: PlayedScore;
|
|
11
|
+
readonly speech: Speech | null;
|
|
12
|
+
}
|
|
13
|
+
/** A film's sound as recorded: the score and decoded narration in film time, and how film time is played. */
|
|
14
|
+
export interface FilmSoundInput {
|
|
15
|
+
readonly score: ScoreData | null;
|
|
16
|
+
readonly narration: DecodedNarration | null;
|
|
17
|
+
readonly timeline: FilmTimeline;
|
|
18
|
+
}
|
|
19
|
+
/** Places the score and the narration in play time. */
|
|
20
|
+
export declare function filmSound(input: FilmSoundInput): FilmSound;
|
|
21
|
+
export interface FilmAudioRender extends LoudnessMeasurement {
|
|
22
|
+
readonly buffer: AudioBuffer;
|
|
23
|
+
/** Sounds scheduled. */
|
|
24
|
+
readonly voices: number;
|
|
25
|
+
/** Milliseconds the render took. */
|
|
26
|
+
readonly ms: number;
|
|
27
|
+
/** The score's loudness before normalization. */
|
|
28
|
+
readonly lufsRaw: number;
|
|
29
|
+
/** The normalization gain applied to the score. */
|
|
30
|
+
readonly gain: number;
|
|
31
|
+
}
|
|
32
|
+
export interface RenderOptions {
|
|
33
|
+
readonly sampleRate?: number;
|
|
34
|
+
/** Seconds rendered past the film's end. */
|
|
35
|
+
readonly tail?: number;
|
|
36
|
+
}
|
|
37
|
+
export declare const DEFAULT_SAMPLE_RATE = 48000;
|
|
38
|
+
/** Rendered ahead of play time 0 and dropped, so a sound heard before its onset (a reverse swell) is whole. */
|
|
39
|
+
export declare const OFFLINE_PREROLL_SECONDS = 0.3;
|
|
40
|
+
/** Narration's loudness is measured on its first minute of takes. */
|
|
41
|
+
export declare const SPEECH_MEASURE_SECONDS = 60;
|
|
42
|
+
/** A score renders a stretch at a time: the context stops this often, in context seconds, for the next sounds. */
|
|
43
|
+
export declare const OFFLINE_RENDER_CHUNK_SECONDS = 2;
|
|
44
|
+
/** A chunked render that reaches no stop for this long is given up and the score rendered whole. */
|
|
45
|
+
export declare const OFFLINE_CHUNK_STALL_MS = 30000;
|
|
46
|
+
/** What a chunked render uses of its context. */
|
|
47
|
+
export type ChunkedContext = Pick<OfflineAudioContext, 'currentTime' | 'length' | 'sampleRate' | 'createGain' | 'suspend' | 'resume' | 'startRendering'>;
|
|
48
|
+
/** Builds sound `index` on `bus`; false when nothing was built. */
|
|
49
|
+
export type ScheduleHeard = (bus: VoiceBus, index: number) => boolean;
|
|
50
|
+
/** A finished render and the sounds scheduled for it. */
|
|
51
|
+
export interface RenderedPass {
|
|
52
|
+
readonly rendered: AudioBuffer;
|
|
53
|
+
readonly voices: number;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Renders the sounds a stretch at a time, so a long score never holds the nodes of all its sounds at once. The context
|
|
57
|
+
* stops every {@link OFFLINE_RENDER_CHUNK_SECONDS}; at each stop the sounds heard from before the stop after next are
|
|
58
|
+
* built on a fresh pair of gains, which count their sources, and a pair silent at two stops in a row is let go; the
|
|
59
|
+
* last stretch takes every sound left. `sounds` come ordered by start, `contextTime` gives a play time's context time.
|
|
60
|
+
* Null when the render must be done whole instead: a stop came more than half a stretch late, none came for
|
|
61
|
+
* {@link OFFLINE_CHUNK_STALL_MS}, or sounds were left unscheduled at the end.
|
|
62
|
+
*/
|
|
63
|
+
export declare function renderInChunks(context: ChunkedContext, bus: VoiceBus, sounds: readonly HeardSpan[], contextTime: (play: number) => number, schedule: ScheduleHeard): Promise<RenderedPass | null>;
|
|
64
|
+
/** The normalization gain already known for a score, if any. */
|
|
65
|
+
export declare function knownScoreGain(score: PlayedScore): number | undefined;
|
|
66
|
+
/** Renders the score alone, measures it, and gives the gain that brings it to its loudness target. */
|
|
67
|
+
export declare function measureScoreGain(score: PlayedScore, sampleRate?: number): Promise<number>;
|
|
68
|
+
/**
|
|
69
|
+
* The whole film's sound, for export: the score normalized to its loudness target (rendered and measured first when its
|
|
70
|
+
* gain is not known yet), with the narration on top.
|
|
71
|
+
*/
|
|
72
|
+
export declare function renderFilmAudio(sound: FilmSound, options?: RenderOptions): Promise<FilmAudioRender>;
|
|
73
|
+
/** Lays takes end to end, up to the first minute, as narration's loudness is measured on them. */
|
|
74
|
+
export declare function speechForMeasure(takes: Iterable<AudioBuffer>): {
|
|
75
|
+
clips: SpeechClip[];
|
|
76
|
+
duration: PlaySeconds;
|
|
77
|
+
};
|
|
78
|
+
/** The gain that brings narration's takes, played through the mix, to `targetLufs`. */
|
|
79
|
+
export declare function measureSpeechGain(takes: Iterable<AudioBuffer>, targetLufs: number, sampleRate?: number): Promise<number>;
|
package/src/offline.js
ADDED
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import { playSeconds } from '@open-artifact/motion-core';
|
|
2
|
+
import { loudnessGain, measureIntegratedLoudness } from "./loudness.js";
|
|
3
|
+
import { createMasterBus } from "./mixer.js";
|
|
4
|
+
import { placeSpeech } from "./narration/speech.js";
|
|
5
|
+
import { playScore } from "./played-score.js";
|
|
6
|
+
import { createScoreSession } from "./session.js";
|
|
7
|
+
import { scheduleSound } from "./synth/schedule.js";
|
|
8
|
+
import { heardFrom } from "./window.js";
|
|
9
|
+
/** Places the score and the narration in play time. */
|
|
10
|
+
export function filmSound(input) {
|
|
11
|
+
return { score: playScore(input.score, input.timeline), speech: placeSpeech(input.narration, input.timeline) };
|
|
12
|
+
}
|
|
13
|
+
export const DEFAULT_SAMPLE_RATE = 48000;
|
|
14
|
+
/** Rendered ahead of play time 0 and dropped, so a sound heard before its onset (a reverse swell) is whole. */
|
|
15
|
+
export const OFFLINE_PREROLL_SECONDS = 0.3;
|
|
16
|
+
/** Narration's loudness is measured on its first minute of takes. */
|
|
17
|
+
export const SPEECH_MEASURE_SECONDS = 60;
|
|
18
|
+
/** A normalization this close to unity is not worth a second render. */
|
|
19
|
+
const UNITY_TOLERANCE_DB = 0.5;
|
|
20
|
+
/** A score renders a stretch at a time: the context stops this often, in context seconds, for the next sounds. */
|
|
21
|
+
export const OFFLINE_RENDER_CHUNK_SECONDS = 2;
|
|
22
|
+
/** A chunked render that reaches no stop for this long is given up and the score rendered whole. */
|
|
23
|
+
export const OFFLINE_CHUNK_STALL_MS = 30_000;
|
|
24
|
+
function offlineContext(channels, frames, sampleRate) {
|
|
25
|
+
if (typeof OfflineAudioContext === 'undefined')
|
|
26
|
+
throw new Error('This browser cannot render audio offline.');
|
|
27
|
+
return new OfflineAudioContext(channels, frames, sampleRate);
|
|
28
|
+
}
|
|
29
|
+
/** Every sound heard from play time 0 scheduled up front, then the whole render. */
|
|
30
|
+
async function renderWhole(context, bus, sounds, schedule) {
|
|
31
|
+
let voices = 0;
|
|
32
|
+
for (const index of heardFrom(sounds, playSeconds(0)))
|
|
33
|
+
if (schedule(bus, index))
|
|
34
|
+
voices++;
|
|
35
|
+
return { rendered: await context.startRendering(), voices };
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Renders the sounds a stretch at a time, so a long score never holds the nodes of all its sounds at once. The context
|
|
39
|
+
* stops every {@link OFFLINE_RENDER_CHUNK_SECONDS}; at each stop the sounds heard from before the stop after next are
|
|
40
|
+
* built on a fresh pair of gains, which count their sources, and a pair silent at two stops in a row is let go; the
|
|
41
|
+
* last stretch takes every sound left. `sounds` come ordered by start, `contextTime` gives a play time's context time.
|
|
42
|
+
* Null when the render must be done whole instead: a stop came more than half a stretch late, none came for
|
|
43
|
+
* {@link OFFLINE_CHUNK_STALL_MS}, or sounds were left unscheduled at the end.
|
|
44
|
+
*/
|
|
45
|
+
export function renderInChunks(context, bus, sounds, contextTime, schedule) {
|
|
46
|
+
const heard = heardFrom(sounds, playSeconds(0));
|
|
47
|
+
const length = context.length / context.sampleRate;
|
|
48
|
+
// The stretches' gains come and go: the sends stay stereo whatever is connected to them.
|
|
49
|
+
for (const send of [bus.dry, bus.reverb]) {
|
|
50
|
+
send.channelCount = 2;
|
|
51
|
+
send.channelCountMode = 'explicit';
|
|
52
|
+
}
|
|
53
|
+
let stretches = [];
|
|
54
|
+
let next = 0;
|
|
55
|
+
let voices = 0;
|
|
56
|
+
let stall;
|
|
57
|
+
let giveUp = () => { };
|
|
58
|
+
const gaveUp = new Promise((resolve) => {
|
|
59
|
+
giveUp = () => resolve(null);
|
|
60
|
+
});
|
|
61
|
+
const scheduleStretch = (from) => {
|
|
62
|
+
if (context.currentTime > from + OFFLINE_RENDER_CHUNK_SECONDS / 2)
|
|
63
|
+
throw new Error('the render stopped too late for the next sounds');
|
|
64
|
+
clearTimeout(stall);
|
|
65
|
+
stall = setTimeout(giveUp, OFFLINE_CHUNK_STALL_MS);
|
|
66
|
+
for (const stretch of stretches) {
|
|
67
|
+
if (!stretch.quiet)
|
|
68
|
+
continue;
|
|
69
|
+
stretch.dry.disconnect();
|
|
70
|
+
stretch.reverb.disconnect();
|
|
71
|
+
}
|
|
72
|
+
stretches = stretches.filter((stretch) => !stretch.quiet);
|
|
73
|
+
for (const stretch of stretches)
|
|
74
|
+
stretch.quiet = stretch.sounding === 0;
|
|
75
|
+
const until = from + 2 * OFFLINE_RENDER_CHUNK_SECONDS;
|
|
76
|
+
const last = until >= length;
|
|
77
|
+
const stretch = { dry: context.createGain(), reverb: context.createGain(), sounding: 0, quiet: false };
|
|
78
|
+
stretch.dry.connect(bus.dry);
|
|
79
|
+
stretch.reverb.connect(bus.reverb);
|
|
80
|
+
stretches.push(stretch);
|
|
81
|
+
const stretchBus = {
|
|
82
|
+
...bus,
|
|
83
|
+
dry: stretch.dry,
|
|
84
|
+
reverb: stretch.reverb,
|
|
85
|
+
track(source) {
|
|
86
|
+
bus.track(source);
|
|
87
|
+
stretch.sounding++;
|
|
88
|
+
source.addEventListener('ended', () => stretch.sounding--);
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
for (; next < heard.length; next++) {
|
|
92
|
+
const index = heard[next];
|
|
93
|
+
if (!last && contextTime(sounds[index].start) >= until)
|
|
94
|
+
break;
|
|
95
|
+
if (schedule(stretchBus, index))
|
|
96
|
+
voices++;
|
|
97
|
+
}
|
|
98
|
+
if (last)
|
|
99
|
+
return;
|
|
100
|
+
const stop = from + OFFLINE_RENDER_CHUNK_SECONDS;
|
|
101
|
+
context
|
|
102
|
+
.suspend(stop)
|
|
103
|
+
.then(() => scheduleStretch(stop))
|
|
104
|
+
.catch(giveUp)
|
|
105
|
+
.then(() => context.resume())
|
|
106
|
+
.catch(giveUp);
|
|
107
|
+
};
|
|
108
|
+
scheduleStretch(0);
|
|
109
|
+
return Promise.race([context.startRendering().then((rendered) => (next < heard.length ? null : { rendered, voices })), gaveUp]).finally(() => clearTimeout(stall));
|
|
110
|
+
}
|
|
111
|
+
async function renderPass(sound, renderSeconds, sampleRate, normalization, withSpeech, inChunks = true) {
|
|
112
|
+
const prerollFrames = Math.round(sampleRate * OFFLINE_PREROLL_SECONDS);
|
|
113
|
+
const frames = Math.ceil(sampleRate * renderSeconds);
|
|
114
|
+
const context = offlineContext(2, prerollFrames + frames, sampleRate);
|
|
115
|
+
const output = context.createBuffer(2, frames, sampleRate);
|
|
116
|
+
const master = createMasterBus(context);
|
|
117
|
+
master.normalization.gain.value = normalization;
|
|
118
|
+
const session = createScoreSession(master, {
|
|
119
|
+
master: sound.score.master,
|
|
120
|
+
speech: withSpeech ? sound.speech : null,
|
|
121
|
+
duration: sound.score.duration,
|
|
122
|
+
origin: prerollFrames / sampleRate,
|
|
123
|
+
rate: 1,
|
|
124
|
+
});
|
|
125
|
+
// The narration's takes are few and start up front; the score's sounds are scheduled by stretch where they can be.
|
|
126
|
+
if (withSpeech && sound.speech !== null)
|
|
127
|
+
session.speak(playSeconds(0));
|
|
128
|
+
const { sounds } = sound.score;
|
|
129
|
+
const schedule = (bus, index) => {
|
|
130
|
+
const heard = sounds[index];
|
|
131
|
+
return scheduleSound(bus, heard.patch, heard.sound, session.contextTime(heard.at));
|
|
132
|
+
};
|
|
133
|
+
const pass = inChunks && sounds.length > 0 && typeof context.suspend === 'function'
|
|
134
|
+
? await renderInChunks(context, session.bus, sounds, session.contextTime, schedule)
|
|
135
|
+
: await renderWhole(context, session.bus, sounds, schedule);
|
|
136
|
+
if (pass === null)
|
|
137
|
+
return renderPass(sound, renderSeconds, sampleRate, normalization, withSpeech, false);
|
|
138
|
+
for (let channel = 0; channel < 2; channel++)
|
|
139
|
+
output.copyToChannel(pass.rendered.getChannelData(channel).subarray(prerollFrames), channel);
|
|
140
|
+
return { buffer: output, voices: pass.voices };
|
|
141
|
+
}
|
|
142
|
+
/** A score's normalization gain is kept per score object, and by content for the newest few (a reload rebuilds the same score). */
|
|
143
|
+
const scoreGains = new WeakMap();
|
|
144
|
+
const contentGains = new Map();
|
|
145
|
+
const CONTENT_GAINS_KEPT = 4;
|
|
146
|
+
const CONTENT_KEY_MAX_CHARS = 4_000_000;
|
|
147
|
+
function contentKey(score, sampleRate) {
|
|
148
|
+
try {
|
|
149
|
+
const key = JSON.stringify([score.sounds, score.master, score.duration, sampleRate]);
|
|
150
|
+
return key.length <= CONTENT_KEY_MAX_CHARS ? key : undefined;
|
|
151
|
+
}
|
|
152
|
+
catch {
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
function rememberGain(score, key, gain) {
|
|
157
|
+
if (gain === undefined)
|
|
158
|
+
return;
|
|
159
|
+
scoreGains.set(score, gain);
|
|
160
|
+
if (key === undefined)
|
|
161
|
+
return;
|
|
162
|
+
contentGains.delete(key);
|
|
163
|
+
contentGains.set(key, gain);
|
|
164
|
+
for (const oldest of contentGains.keys()) {
|
|
165
|
+
if (contentGains.size <= CONTENT_GAINS_KEPT)
|
|
166
|
+
break;
|
|
167
|
+
contentGains.delete(oldest);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
function keptGain(score, key) {
|
|
171
|
+
const gain = key === undefined ? undefined : contentGains.get(key);
|
|
172
|
+
if (gain !== undefined)
|
|
173
|
+
rememberGain(score, key, gain);
|
|
174
|
+
return gain;
|
|
175
|
+
}
|
|
176
|
+
/** The normalization gain already known for a score, if any. */
|
|
177
|
+
export function knownScoreGain(score) {
|
|
178
|
+
return scoreGains.get(score) ?? (score.sounds.length === 0 ? 1 : undefined);
|
|
179
|
+
}
|
|
180
|
+
/** Renders the score alone, measures it, and gives the gain that brings it to its loudness target. */
|
|
181
|
+
export async function measureScoreGain(score, sampleRate = DEFAULT_SAMPLE_RATE) {
|
|
182
|
+
const known = scoreGains.get(score);
|
|
183
|
+
if (known !== undefined)
|
|
184
|
+
return known;
|
|
185
|
+
if (score.sounds.length === 0 || score.master.loudness === null)
|
|
186
|
+
return 1;
|
|
187
|
+
const key = contentKey(score, sampleRate);
|
|
188
|
+
const kept = keptGain(score, key);
|
|
189
|
+
if (kept !== undefined)
|
|
190
|
+
return kept;
|
|
191
|
+
const { buffer } = await renderPass({ score, speech: null }, Math.max(0.1, score.duration), sampleRate, 1, false);
|
|
192
|
+
const measurement = measureIntegratedLoudness(buffer);
|
|
193
|
+
const gain = loudnessGain(measurement, score.master.loudness);
|
|
194
|
+
rememberGain(score, key, Number.isFinite(measurement.lufs) ? gain : undefined);
|
|
195
|
+
return gain;
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* The whole film's sound, for export: the score normalized to its loudness target (rendered and measured first when its
|
|
199
|
+
* gain is not known yet), with the narration on top.
|
|
200
|
+
*/
|
|
201
|
+
export async function renderFilmAudio(sound, options = {}) {
|
|
202
|
+
const sampleRate = options.sampleRate ?? DEFAULT_SAMPLE_RATE;
|
|
203
|
+
const tail = options.tail ?? 0;
|
|
204
|
+
const renderSeconds = Math.max(0.1, sound.score.duration + tail);
|
|
205
|
+
const startedAt = performance.now();
|
|
206
|
+
const known = knownScoreGain(sound.score);
|
|
207
|
+
const key = known === undefined ? contentKey(sound.score, sampleRate) : undefined;
|
|
208
|
+
const gainBefore = known ?? keptGain(sound.score, key);
|
|
209
|
+
const raw = await renderPass(sound, renderSeconds, sampleRate, gainBefore ?? 1, gainBefore !== undefined);
|
|
210
|
+
const rawMeasurement = measureIntegratedLoudness(raw.buffer);
|
|
211
|
+
if (gainBefore !== undefined) {
|
|
212
|
+
return {
|
|
213
|
+
buffer: raw.buffer,
|
|
214
|
+
voices: raw.voices,
|
|
215
|
+
ms: performance.now() - startedAt,
|
|
216
|
+
lufsRaw: rawMeasurement.lufs - 20 * Math.log10(gainBefore),
|
|
217
|
+
gain: gainBefore,
|
|
218
|
+
...rawMeasurement,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
const gain = loudnessGain(rawMeasurement, sound.score.master.loudness);
|
|
222
|
+
// A render with a tail measures more than the score, so its gain is not kept for the score.
|
|
223
|
+
rememberGain(sound.score, key, tail === 0 && Number.isFinite(rawMeasurement.lufs) ? gain : undefined);
|
|
224
|
+
const final = sound.speech === null && Math.abs(20 * Math.log10(gain)) < UNITY_TOLERANCE_DB
|
|
225
|
+
? raw
|
|
226
|
+
: await renderPass(sound, renderSeconds, sampleRate, gain, true);
|
|
227
|
+
const measurement = final === raw ? rawMeasurement : measureIntegratedLoudness(final.buffer);
|
|
228
|
+
return {
|
|
229
|
+
buffer: final.buffer,
|
|
230
|
+
voices: final.voices,
|
|
231
|
+
ms: performance.now() - startedAt,
|
|
232
|
+
lufsRaw: rawMeasurement.lufs,
|
|
233
|
+
gain,
|
|
234
|
+
...measurement,
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
/** Lays takes end to end, up to the first minute, as narration's loudness is measured on them. */
|
|
238
|
+
export function speechForMeasure(takes) {
|
|
239
|
+
const clips = [];
|
|
240
|
+
let duration = 0;
|
|
241
|
+
for (const buffer of takes) {
|
|
242
|
+
if (duration >= SPEECH_MEASURE_SECONDS)
|
|
243
|
+
break;
|
|
244
|
+
clips.push({ at: playSeconds(duration), buffer });
|
|
245
|
+
duration += buffer.duration;
|
|
246
|
+
}
|
|
247
|
+
return { clips, duration: playSeconds(duration) };
|
|
248
|
+
}
|
|
249
|
+
/** The gain that brings narration's takes, played through the mix, to `targetLufs`. */
|
|
250
|
+
export async function measureSpeechGain(takes, targetLufs, sampleRate = DEFAULT_SAMPLE_RATE) {
|
|
251
|
+
const { clips, duration } = speechForMeasure(takes);
|
|
252
|
+
if (clips.length === 0)
|
|
253
|
+
return 1;
|
|
254
|
+
const master = { gain: 0, fadeOut: 0 };
|
|
255
|
+
const render = await renderFilmAudio({ score: { sounds: [], master, duration }, speech: { clips, gain: 1, duck: [] } }, { sampleRate });
|
|
256
|
+
return loudnessGain(render, targetLufs);
|
|
257
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { PlaySeconds } from '@open-artifact/motion-core';
|
|
2
|
+
/** Milliseconds that stand still while the page is hidden, so a film does not jump ahead when it comes back. */
|
|
3
|
+
export interface VisibleClock {
|
|
4
|
+
now(): number;
|
|
5
|
+
/** How many times the page came back into view. */
|
|
6
|
+
returns(): number;
|
|
7
|
+
dispose(): void;
|
|
8
|
+
}
|
|
9
|
+
export declare function createVisibleClock(): VisibleClock;
|
|
10
|
+
/** Where the playhead is while playing. */
|
|
11
|
+
export interface PlaybackTimeSource {
|
|
12
|
+
now(): PlaySeconds;
|
|
13
|
+
/** True when the system cut audible sound off (a call on a phone): the player should pause. */
|
|
14
|
+
interrupted(): boolean;
|
|
15
|
+
}
|
|
16
|
+
/** Play time from the visible clock alone, for a page without audio. */
|
|
17
|
+
export declare function wallClockSource(from: PlaySeconds, rate: number, clock: VisibleClock): PlaybackTimeSource;
|
|
18
|
+
/** Audio that has not moved for this long is taken as stalled, and the picture runs on the visible clock. */
|
|
19
|
+
export declare const STALL_SECONDS = 0.3;
|
|
20
|
+
export interface AudioClockOptions {
|
|
21
|
+
readonly context: BaseAudioContext;
|
|
22
|
+
readonly clock: VisibleClock;
|
|
23
|
+
readonly from: PlaySeconds;
|
|
24
|
+
readonly rate: number;
|
|
25
|
+
/** The context time play time 0 stands at in the current pass; it moves when the pass is restarted after a stall. */
|
|
26
|
+
origin(): number;
|
|
27
|
+
/** Whether sound would be heard now (playing, not muted, something to hear). */
|
|
28
|
+
audible(): boolean;
|
|
29
|
+
/** Audio stalled: the pass should stop until it moves again. */
|
|
30
|
+
onStall(): void;
|
|
31
|
+
/** Audio moves again: the pass should restart from `at`. */
|
|
32
|
+
onResume(at: PlaySeconds): void;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Play time read from the audio context, so picture and sound stay together. When the context stops moving (a hidden
|
|
36
|
+
* tab, a suspended device), time runs on the visible clock instead, and the sound restarts where the picture is once
|
|
37
|
+
* the context moves again. Time never runs backwards.
|
|
38
|
+
*/
|
|
39
|
+
export declare function audioClockSource(options: AudioClockOptions): PlaybackTimeSource;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
import { playSeconds } from '@open-artifact/motion-core';
|
|
2
|
+
import { playTimeAt } from "./window.js";
|
|
3
|
+
export function createVisibleClock() {
|
|
4
|
+
if (typeof document === 'undefined')
|
|
5
|
+
return { now: () => performance.now(), returns: () => 0, dispose: () => undefined };
|
|
6
|
+
let hiddenFor = 0;
|
|
7
|
+
let returns = 0;
|
|
8
|
+
let hiddenSince = document.hidden ? performance.now() : null;
|
|
9
|
+
const update = () => {
|
|
10
|
+
if (document.hidden)
|
|
11
|
+
hiddenSince ??= performance.now();
|
|
12
|
+
else if (hiddenSince !== null) {
|
|
13
|
+
hiddenFor += performance.now() - hiddenSince;
|
|
14
|
+
hiddenSince = null;
|
|
15
|
+
returns++;
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
document.addEventListener('visibilitychange', update);
|
|
19
|
+
return {
|
|
20
|
+
now: () => (hiddenSince ?? performance.now()) - hiddenFor,
|
|
21
|
+
returns: () => returns,
|
|
22
|
+
dispose() {
|
|
23
|
+
document.removeEventListener('visibilitychange', update);
|
|
24
|
+
if (hiddenSince !== null)
|
|
25
|
+
hiddenFor += performance.now() - hiddenSince;
|
|
26
|
+
hiddenSince = null;
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
/** Play time from the visible clock alone, for a page without audio. */
|
|
31
|
+
export function wallClockSource(from, rate, clock) {
|
|
32
|
+
const startedAt = clock.now();
|
|
33
|
+
return {
|
|
34
|
+
now: () => playSeconds(from + ((clock.now() - startedAt) / 1000) * rate),
|
|
35
|
+
interrupted: () => false,
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
/** Audio that has not moved for this long is taken as stalled, and the picture runs on the visible clock. */
|
|
39
|
+
export const STALL_SECONDS = 0.3;
|
|
40
|
+
/**
|
|
41
|
+
* Play time read from the audio context, so picture and sound stay together. When the context stops moving (a hidden
|
|
42
|
+
* tab, a suspended device), time runs on the visible clock instead, and the sound restarts where the picture is once
|
|
43
|
+
* the context moves again. Time never runs backwards.
|
|
44
|
+
*/
|
|
45
|
+
export function audioClockSource(options) {
|
|
46
|
+
const { context, clock, from, rate } = options;
|
|
47
|
+
const startedAt = clock.now();
|
|
48
|
+
let lastAudioTime = context.currentTime;
|
|
49
|
+
let lastMovedAt = 0;
|
|
50
|
+
let stalled = false;
|
|
51
|
+
let everMoved = false;
|
|
52
|
+
let audibleStall = false;
|
|
53
|
+
let seenReturns = clock.returns();
|
|
54
|
+
let position = 0;
|
|
55
|
+
let stallBase = 0;
|
|
56
|
+
let stallWall = 0;
|
|
57
|
+
return {
|
|
58
|
+
now() {
|
|
59
|
+
const wall = (clock.now() - startedAt) / 1000;
|
|
60
|
+
const moved = context.currentTime - lastAudioTime > 0.001;
|
|
61
|
+
if (moved) {
|
|
62
|
+
everMoved = true;
|
|
63
|
+
lastAudioTime = context.currentTime;
|
|
64
|
+
}
|
|
65
|
+
const suspended = everMoved && context.state === 'suspended';
|
|
66
|
+
if (moved || suspended || seenReturns !== clock.returns())
|
|
67
|
+
lastMovedAt = wall;
|
|
68
|
+
seenReturns = clock.returns();
|
|
69
|
+
if (!stalled && wall - lastMovedAt > STALL_SECONDS) {
|
|
70
|
+
stalled = true;
|
|
71
|
+
audibleStall = everMoved && options.audible();
|
|
72
|
+
stallBase = position;
|
|
73
|
+
stallWall = wall;
|
|
74
|
+
options.onStall();
|
|
75
|
+
}
|
|
76
|
+
else if (stalled && moved) {
|
|
77
|
+
stalled = false;
|
|
78
|
+
audibleStall = false;
|
|
79
|
+
options.onResume(playSeconds(from + position));
|
|
80
|
+
}
|
|
81
|
+
const elapsed = stalled
|
|
82
|
+
? stallBase + (wall - stallWall) * rate
|
|
83
|
+
: Math.max(0, playTimeAt(context.currentTime, options.origin(), rate) - from);
|
|
84
|
+
if (elapsed > position)
|
|
85
|
+
position = elapsed;
|
|
86
|
+
return playSeconds(from + position);
|
|
87
|
+
},
|
|
88
|
+
interrupted: () => audibleStall && String(context.state) === 'interrupted',
|
|
89
|
+
};
|
|
90
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { PlaySeconds, ScoreData, ScoreMaster } from '@open-artifact/motion-core';
|
|
2
|
+
import type { ResolvedPatch } from './synth/presets.ts';
|
|
3
|
+
import type { Sound } from './synth/voice.ts';
|
|
4
|
+
import type { FilmTimeline } from './timeline.ts';
|
|
5
|
+
/** Score events later than this are not played. */
|
|
6
|
+
export declare const MAX_EVENT_SECONDS = 900;
|
|
7
|
+
/** A score sound placed in play time: its onset, and the span it is heard over (its ring included). */
|
|
8
|
+
export interface PlayedSound {
|
|
9
|
+
readonly patch: ResolvedPatch;
|
|
10
|
+
readonly sound: Sound;
|
|
11
|
+
readonly at: PlaySeconds;
|
|
12
|
+
readonly start: PlaySeconds;
|
|
13
|
+
readonly end: PlaySeconds;
|
|
14
|
+
}
|
|
15
|
+
/** A score as it is heard: its sounds in play time, ordered by when they start to be heard. */
|
|
16
|
+
export interface PlayedScore {
|
|
17
|
+
readonly sounds: readonly PlayedSound[];
|
|
18
|
+
readonly master: ScoreMaster;
|
|
19
|
+
readonly duration: PlaySeconds;
|
|
20
|
+
}
|
|
21
|
+
/** What a film without a score plays: nothing, at the default master. */
|
|
22
|
+
export declare const SILENT_MASTER: ScoreMaster;
|
|
23
|
+
/**
|
|
24
|
+
* Places the score's events in play time. Starts and lengths both move with the timeline, so a stretched span's sound
|
|
25
|
+
* slows with its picture at the same pitch; an event whose onset is cut out is not played, and one whose end is cut out
|
|
26
|
+
* keeps its written length.
|
|
27
|
+
*/
|
|
28
|
+
export declare function playScore(score: ScoreData | null, timeline: FilmTimeline): PlayedScore;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { filmSeconds, playSeconds } from '@open-artifact/motion-core';
|
|
2
|
+
import { soundExtent } from "./synth/presets.js";
|
|
3
|
+
import { MAX_SOUND_SECONDS, voiceOf } from "./synth/voice.js";
|
|
4
|
+
/** Score events later than this are not played. */
|
|
5
|
+
export const MAX_EVENT_SECONDS = 900;
|
|
6
|
+
/** What a film without a score plays: nothing, at the default master. */
|
|
7
|
+
export const SILENT_MASTER = { gain: -6, fadeOut: 1 };
|
|
8
|
+
/**
|
|
9
|
+
* Places the score's events in play time. Starts and lengths both move with the timeline, so a stretched span's sound
|
|
10
|
+
* slows with its picture at the same pitch; an event whose onset is cut out is not played, and one whose end is cut out
|
|
11
|
+
* keeps its written length.
|
|
12
|
+
*/
|
|
13
|
+
export function playScore(score, timeline) {
|
|
14
|
+
const sounds = [];
|
|
15
|
+
for (const event of score?.events ?? []) {
|
|
16
|
+
if (!Number.isFinite(event.t) || !Number.isFinite(event.dur) || event.t < 0 || event.t >= MAX_EVENT_SECONDS)
|
|
17
|
+
continue;
|
|
18
|
+
const voice = voiceOf(event);
|
|
19
|
+
if (voice === null)
|
|
20
|
+
continue;
|
|
21
|
+
const at = timeline.toPlay(voice.t);
|
|
22
|
+
if (at === null)
|
|
23
|
+
continue;
|
|
24
|
+
const endAt = timeline.toPlay(filmSeconds(voice.t + voice.sound.dur));
|
|
25
|
+
const dur = Math.max(0.01, Math.min(MAX_SOUND_SECONDS, endAt === null ? voice.sound.dur : endAt - at));
|
|
26
|
+
const [start, end] = soundExtent(voice.patch, at, dur);
|
|
27
|
+
sounds.push({ patch: voice.patch, sound: { ...voice.sound, dur }, at, start: playSeconds(start), end: playSeconds(end) });
|
|
28
|
+
}
|
|
29
|
+
sounds.sort((first, second) => first.start - second.start);
|
|
30
|
+
return { sounds, master: score?.master ?? SILENT_MASTER, duration: timeline.playDuration };
|
|
31
|
+
}
|
package/src/player.d.ts
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { PlaySeconds } from '@open-artifact/motion-core';
|
|
2
|
+
import type { FilmSound, FilmSoundInput } from './offline.ts';
|
|
3
|
+
import type { VisibleClock } from './playback-clock.ts';
|
|
4
|
+
import type { SchedulerTimers } from './scheduler.ts';
|
|
5
|
+
export interface SoundPlayerState {
|
|
6
|
+
/** An audio context exists and is not closed: sound can be heard once unmuted. */
|
|
7
|
+
readonly available: boolean;
|
|
8
|
+
readonly playing: boolean;
|
|
9
|
+
readonly muted: boolean;
|
|
10
|
+
/** 0…1. */
|
|
11
|
+
readonly volume: number;
|
|
12
|
+
/** Play seconds per second. */
|
|
13
|
+
readonly rate: number;
|
|
14
|
+
/** The system cut the sound off while playing; the page should pause. */
|
|
15
|
+
readonly interrupted: boolean;
|
|
16
|
+
}
|
|
17
|
+
export interface FilmSoundPlayerOptions {
|
|
18
|
+
/** Makes the page's audio context; by default a 48 kHz interactive AudioContext. Null: no audio here. */
|
|
19
|
+
readonly createContext?: () => AudioContext | null;
|
|
20
|
+
readonly timers?: SchedulerTimers;
|
|
21
|
+
readonly clock?: VisibleClock;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* The film's sound in the editor: the score synthesized live and the narration over it, in step with the picture.
|
|
25
|
+
* Starts muted. The page reads `now()` every frame while playing, so the picture follows the sound.
|
|
26
|
+
*/
|
|
27
|
+
export declare class FilmSoundPlayer {
|
|
28
|
+
#private;
|
|
29
|
+
constructor(options?: FilmSoundPlayerOptions);
|
|
30
|
+
get state(): SoundPlayerState;
|
|
31
|
+
/** The film's sound as last loaded, in play time: what an export renders. */
|
|
32
|
+
get sound(): FilmSound;
|
|
33
|
+
subscribe(listener: (state: SoundPlayerState) => void): () => void;
|
|
34
|
+
/** Takes a new score, narration or timeline; playing on from where the playhead is. */
|
|
35
|
+
load(input: FilmSoundInput): void;
|
|
36
|
+
/** Makes or wakes the audio context; call from a click or key press, which browsers require for sound. */
|
|
37
|
+
prime(): void;
|
|
38
|
+
play(from: PlaySeconds): void;
|
|
39
|
+
pause(): void;
|
|
40
|
+
seek(to: PlaySeconds): void;
|
|
41
|
+
setRate(rate: number): void;
|
|
42
|
+
setMuted(muted: boolean): void;
|
|
43
|
+
setVolume(volume: number): void;
|
|
44
|
+
/** Where the playhead is: from the audio context while it moves, else from the visible clock. */
|
|
45
|
+
now(): PlaySeconds;
|
|
46
|
+
dispose(): void;
|
|
47
|
+
}
|