@open-artifact/motion-audio 0.0.0-stage → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +5 -0
  3. package/README.md +5 -2
  4. package/package.json +36 -4
  5. package/src/index.d.ts +46 -0
  6. package/src/index.js +25 -0
  7. package/src/keepalive.d.ts +10 -0
  8. package/src/keepalive.js +49 -0
  9. package/src/loudness.d.ts +32 -0
  10. package/src/loudness.js +108 -0
  11. package/src/midi.d.ts +16 -0
  12. package/src/midi.js +179 -0
  13. package/src/mixer.d.ts +29 -0
  14. package/src/mixer.js +70 -0
  15. package/src/narration/captions.d.ts +24 -0
  16. package/src/narration/captions.js +45 -0
  17. package/src/narration/decode.d.ts +25 -0
  18. package/src/narration/decode.js +72 -0
  19. package/src/narration/duck.d.ts +20 -0
  20. package/src/narration/duck.js +34 -0
  21. package/src/narration/limits.d.ts +23 -0
  22. package/src/narration/limits.js +16 -0
  23. package/src/narration/peaks.d.ts +14 -0
  24. package/src/narration/peaks.js +27 -0
  25. package/src/narration/plan.d.ts +12 -0
  26. package/src/narration/plan.js +31 -0
  27. package/src/narration/speech.d.ts +31 -0
  28. package/src/narration/speech.js +24 -0
  29. package/src/offline.d.ts +79 -0
  30. package/src/offline.js +257 -0
  31. package/src/playback-clock.d.ts +39 -0
  32. package/src/playback-clock.js +90 -0
  33. package/src/played-score.d.ts +28 -0
  34. package/src/played-score.js +31 -0
  35. package/src/player.d.ts +47 -0
  36. package/src/player.js +237 -0
  37. package/src/reverb.d.ts +10 -0
  38. package/src/reverb.js +38 -0
  39. package/src/scheduler.d.ts +29 -0
  40. package/src/scheduler.js +64 -0
  41. package/src/session.d.ts +24 -0
  42. package/src/session.js +133 -0
  43. package/src/synth/drums.d.ts +4 -0
  44. package/src/synth/drums.js +97 -0
  45. package/src/synth/envelope.d.ts +26 -0
  46. package/src/synth/envelope.js +39 -0
  47. package/src/synth/fx.d.ts +9 -0
  48. package/src/synth/fx.js +131 -0
  49. package/src/synth/nodes.d.ts +22 -0
  50. package/src/synth/nodes.js +63 -0
  51. package/src/synth/noise.d.ts +6 -0
  52. package/src/synth/noise.js +31 -0
  53. package/src/synth/patches.d.ts +5 -0
  54. package/src/synth/patches.js +119 -0
  55. package/src/synth/presets.d.ts +32 -0
  56. package/src/synth/presets.js +136 -0
  57. package/src/synth/schedule.d.ts +5 -0
  58. package/src/synth/schedule.js +25 -0
  59. package/src/synth/voice.d.ts +35 -0
  60. package/src/synth/voice.js +52 -0
  61. package/src/timeline.d.ts +25 -0
  62. package/src/timeline.js +49 -0
  63. package/src/wave.d.ts +17 -0
  64. package/src/wave.js +111 -0
  65. package/src/window.d.ts +32 -0
  66. package/src/window.js +59 -0
@@ -0,0 +1,35 @@
1
+ import type { FilmSeconds, ScoreEvent } from '@open-artifact/motion-core';
2
+ import type { ResolvedPatch } from './presets.ts';
3
+ /** A sound's own settings, each within its bounds. `into` turns a whoosh around; `rate` is a boing's wobble per second. */
4
+ export interface VoiceSettings {
5
+ pan?: number;
6
+ gain?: number;
7
+ rev?: number;
8
+ detune?: number;
9
+ bright?: number;
10
+ into?: number;
11
+ rate?: number;
12
+ }
13
+ /** What a patch needs to sound: its length in seconds as heard, its MIDI pitch and velocity, its settings. */
14
+ export interface Sound {
15
+ readonly dur: number;
16
+ readonly midi: number;
17
+ readonly vel: number;
18
+ readonly settings: VoiceSettings;
19
+ }
20
+ /** A score event made safe to synthesize: its patch resolved, every number within bounds. */
21
+ export interface Voice {
22
+ readonly t: FilmSeconds;
23
+ readonly patch: ResolvedPatch;
24
+ readonly sound: Sound;
25
+ }
26
+ export declare const MAX_SOUND_SECONDS = 30;
27
+ export declare const midiToFrequency: (midi: number) => number;
28
+ export declare const velocityToGain: (velocity: number) => number;
29
+ export declare const decibelsToGain: (decibels: number) => number;
30
+ export declare function bounded(input: unknown, fallback: number, least: number, most: number): number;
31
+ export declare function readVoiceSettings(params: ScoreEvent['params']): VoiceSettings;
32
+ /** Null for a drum or effect nobody knows. */
33
+ export declare function voiceOf(event: ScoreEvent): Voice | null;
34
+ /** The gain a sound enters the mix at: its velocity on a curve, its patch's level and its own. */
35
+ export declare function voiceGain(sound: Sound, presetGainDb: number): number;
@@ -0,0 +1,52 @@
1
+ import { filmSeconds } from '@open-artifact/motion-core';
2
+ import { resolvePatch } from "./presets.js";
3
+ const SETTING_BOUNDS = {
4
+ pan: [-1, 1],
5
+ gain: [-60, 12],
6
+ rev: [0, 1],
7
+ detune: [-2400, 2400],
8
+ bright: [0, 2],
9
+ into: [0, 1],
10
+ rate: [0.5, 40],
11
+ };
12
+ const SETTING_KEYS = Object.keys(SETTING_BOUNDS);
13
+ export const MAX_SOUND_SECONDS = 30;
14
+ export const midiToFrequency = (midi) => 440 * Math.pow(2, (midi - 69) / 12);
15
+ export const velocityToGain = (velocity) => Math.pow(Math.max(1, Math.min(127, velocity)) / 127, 1.5);
16
+ export const decibelsToGain = (decibels) => Math.pow(10, decibels / 20);
17
+ export function bounded(input, fallback, least, most) {
18
+ return typeof input === 'number' && Number.isFinite(input) ? Math.max(least, Math.min(most, input)) : fallback;
19
+ }
20
+ export function readVoiceSettings(params) {
21
+ const settings = {};
22
+ if (params === undefined || params === null || typeof params !== 'object')
23
+ return settings;
24
+ const raw = { ...params };
25
+ for (const key of SETTING_KEYS) {
26
+ const value = raw[key];
27
+ const [least, most] = SETTING_BOUNDS[key];
28
+ if (typeof value === 'number' && Number.isFinite(value))
29
+ settings[key] = Math.max(least, Math.min(most, value));
30
+ }
31
+ return settings;
32
+ }
33
+ /** Null for a drum or effect nobody knows. */
34
+ export function voiceOf(event) {
35
+ const patch = resolvePatch(event.kind, event.patch);
36
+ if (patch === null)
37
+ return null;
38
+ return {
39
+ t: filmSeconds(bounded(event.t, 0, 0, 1e4)),
40
+ patch,
41
+ sound: {
42
+ dur: bounded(event.dur, 0.25, 0.01, MAX_SOUND_SECONDS),
43
+ midi: Math.round(bounded(event.midi, 60, 0, 127)),
44
+ vel: bounded(event.vel, 64, 1, 127),
45
+ settings: readVoiceSettings(event.params),
46
+ },
47
+ };
48
+ }
49
+ /** The gain a sound enters the mix at: its velocity on a curve, its patch's level and its own. */
50
+ export function voiceGain(sound, presetGainDb) {
51
+ return velocityToGain(sound.vel) * decibelsToGain(presetGainDb + (sound.settings.gain ?? 0));
52
+ }
@@ -0,0 +1,25 @@
1
+ import type { FilmSeconds, PlaySeconds } from '@open-artifact/motion-core';
2
+ /**
3
+ * How the film's own time is played. Sound is recorded in film time and heard in play time.
4
+ * `toPlay` gives null for a film instant that is not played (a span cut out). Before 0 and past the film's end it runs on
5
+ * at normal speed, so a score's lead-in and tail keep their length.
6
+ */
7
+ export interface FilmTimeline {
8
+ readonly filmDuration: FilmSeconds;
9
+ readonly playDuration: PlaySeconds;
10
+ toPlay(film: FilmSeconds): PlaySeconds | null;
11
+ toFilm(play: PlaySeconds): FilmSeconds;
12
+ }
13
+ /** One stretch of the film played at a constant rate: `filmLength` film seconds shown over `playLength` play seconds. */
14
+ export interface TimelineSegment {
15
+ readonly film: FilmSeconds;
16
+ readonly filmLength: number;
17
+ readonly play: PlaySeconds;
18
+ readonly playLength: number;
19
+ }
20
+ export declare function identityTimeline(duration: FilmSeconds): FilmTimeline;
21
+ /**
22
+ * A piecewise-linear timeline. Segments are in film order and do not overlap; film time between two segments is cut out.
23
+ * With no segment the film plays as it is.
24
+ */
25
+ export declare function segmentTimeline(segments: readonly TimelineSegment[], filmDuration: FilmSeconds): FilmTimeline;
@@ -0,0 +1,49 @@
1
+ import { filmSeconds, playSeconds } from '@open-artifact/motion-core';
2
+ const round6 = (value) => Math.round(value * 1e6) / 1e6;
3
+ /** A segment stops just short of the next one's first instant. */
4
+ const EDGE = 1e-9;
5
+ export function identityTimeline(duration) {
6
+ return {
7
+ filmDuration: duration,
8
+ playDuration: playSeconds(duration),
9
+ toPlay: (film) => playSeconds(film),
10
+ toFilm: (play) => filmSeconds(Math.max(0, Math.min(duration, play))),
11
+ };
12
+ }
13
+ /**
14
+ * A piecewise-linear timeline. Segments are in film order and do not overlap; film time between two segments is cut out.
15
+ * With no segment the film plays as it is.
16
+ */
17
+ export function segmentTimeline(segments, filmDuration) {
18
+ const parts = segments.filter((part) => part.filmLength > 0 && part.playLength > 0);
19
+ const last = parts.at(-1);
20
+ if (last === undefined)
21
+ return identityTimeline(filmDuration);
22
+ const playDuration = playSeconds(round6(last.play + last.playLength));
23
+ const filmEnd = Math.max(filmDuration, last.film + last.filmLength);
24
+ return {
25
+ filmDuration,
26
+ playDuration,
27
+ toPlay(film) {
28
+ if (film < 0)
29
+ return playSeconds(film);
30
+ if (film >= filmEnd)
31
+ return playSeconds(playDuration + (film - filmEnd));
32
+ const part = parts.find((candidate) => film >= candidate.film && film < candidate.film + candidate.filmLength);
33
+ if (part === undefined)
34
+ return null;
35
+ return playSeconds(part.play + (film - part.film) * (part.playLength / part.filmLength));
36
+ },
37
+ toFilm(play) {
38
+ const at = Math.max(0, Math.min(playDuration, play));
39
+ let part = parts[0];
40
+ for (const candidate of parts)
41
+ if (candidate.play <= at)
42
+ part = candidate;
43
+ if (part === undefined)
44
+ return filmSeconds(at);
45
+ const room = part === last ? part.filmLength : part.filmLength - EDGE;
46
+ return filmSeconds(part.film + Math.min(room, (at - part.play) * (part.filmLength / part.playLength)));
47
+ },
48
+ };
49
+ }
package/src/wave.d.ts ADDED
@@ -0,0 +1,17 @@
1
+ import type { PcmSource } from './loudness.ts';
2
+ /** Writes the 44-byte header of a 16-bit PCM wave filling `bytes` (header included). */
3
+ export declare function writeWaveHeader(bytes: ArrayBuffer, sampleRate: number, channels: number): DataView;
4
+ /** A buffer as a 16-bit PCM wave file, its samples clamped to full scale. */
5
+ export declare function encodePcmWave(buffer: PcmSource): Uint8Array<ArrayBuffer>;
6
+ /** A buffer's channels interleaved frame by frame, as encoders take PCM. */
7
+ export declare function interleave(buffer: PcmSource): Float32Array<ArrayBuffer>;
8
+ /** A take that is a plain 16-bit PCM wave: its length, and the same samples behind a bare 44-byte header. */
9
+ export interface PcmWave {
10
+ readonly seconds: number;
11
+ bare(): ArrayBuffer;
12
+ }
13
+ /**
14
+ * Reads a take as a plain 16-bit PCM wave (one or two channels, up to 48 kHz) by its chunks, so its length is known
15
+ * before it is decoded. Anything else is null.
16
+ */
17
+ export declare function parsePcmWave(bytes: ArrayBuffer): PcmWave | null;
package/src/wave.js ADDED
@@ -0,0 +1,111 @@
1
+ const HEADER_BYTES = 44;
2
+ /** A narration take read as a wave may be at most this sample rate. */
3
+ const MOST_TAKE_HZ = 48000;
4
+ /** Writes the 44-byte header of a 16-bit PCM wave filling `bytes` (header included). */
5
+ export function writeWaveHeader(bytes, sampleRate, channels) {
6
+ const view = new DataView(bytes);
7
+ const ascii = (at, text) => {
8
+ for (let index = 0; index < text.length; index++)
9
+ view.setUint8(at + index, text.charCodeAt(index));
10
+ };
11
+ ascii(0, 'RIFF');
12
+ view.setUint32(4, bytes.byteLength - 8, true);
13
+ ascii(8, 'WAVE');
14
+ ascii(12, 'fmt ');
15
+ view.setUint32(16, 16, true);
16
+ view.setUint16(20, 1, true);
17
+ view.setUint16(22, channels, true);
18
+ view.setUint32(24, sampleRate, true);
19
+ view.setUint32(28, sampleRate * channels * 2, true);
20
+ view.setUint16(32, channels * 2, true);
21
+ view.setUint16(34, 16, true);
22
+ ascii(36, 'data');
23
+ view.setUint32(40, bytes.byteLength - HEADER_BYTES, true);
24
+ return view;
25
+ }
26
+ /** A buffer as a 16-bit PCM wave file, its samples clamped to full scale. */
27
+ export function encodePcmWave(buffer) {
28
+ const { numberOfChannels: channels, length } = buffer;
29
+ const bytes = new Uint8Array(HEADER_BYTES + length * channels * 2);
30
+ const view = writeWaveHeader(bytes.buffer, buffer.sampleRate, channels);
31
+ const channelSamples = Array.from({ length: channels }, (_unused, index) => buffer.getChannelData(index));
32
+ let at = HEADER_BYTES;
33
+ for (let frame = 0; frame < length; frame++) {
34
+ for (const samples of channelSamples) {
35
+ const sample = Math.max(-1, Math.min(1, samples[frame] ?? 0));
36
+ view.setInt16(at, sample < 0 ? sample * 32768 : sample * 32767, true);
37
+ at += 2;
38
+ }
39
+ }
40
+ return bytes;
41
+ }
42
+ /** A buffer's channels interleaved frame by frame, as encoders take PCM. */
43
+ export function interleave(buffer) {
44
+ const channels = buffer.numberOfChannels;
45
+ const out = new Float32Array(buffer.length * channels);
46
+ for (let channel = 0; channel < channels; channel++) {
47
+ const samples = buffer.getChannelData(channel);
48
+ for (let frame = 0; frame < buffer.length; frame++)
49
+ out[frame * channels + channel] = samples[frame] ?? 0;
50
+ }
51
+ return out;
52
+ }
53
+ /**
54
+ * Reads a take as a plain 16-bit PCM wave (one or two channels, up to 48 kHz) by its chunks, so its length is known
55
+ * before it is decoded. Anything else is null.
56
+ */
57
+ export function parsePcmWave(bytes) {
58
+ const view = new DataView(bytes);
59
+ const tag = (at) => String.fromCharCode(view.getUint8(at), view.getUint8(at + 1), view.getUint8(at + 2), view.getUint8(at + 3));
60
+ if (bytes.byteLength < 12 || tag(0) !== 'RIFF' || tag(8) !== 'WAVE' || 8 + view.getUint32(4, true) !== bytes.byteLength)
61
+ return null;
62
+ let format = null;
63
+ let data = null;
64
+ for (let at = 12; at < bytes.byteLength;) {
65
+ if (at + 8 > bytes.byteLength)
66
+ return null;
67
+ const size = view.getUint32(at + 4, true);
68
+ if (at + 8 + size > bytes.byteLength)
69
+ return null;
70
+ const chunk = tag(at);
71
+ if (chunk === 'fmt ') {
72
+ if (format !== null || size < 16)
73
+ return null;
74
+ const code = view.getUint16(at + 8, true);
75
+ const channels = view.getUint16(at + 10, true);
76
+ const hz = view.getUint32(at + 12, true);
77
+ const bytesPerSecond = view.getUint32(at + 16, true);
78
+ const blockAlign = view.getUint16(at + 20, true);
79
+ const bits = view.getUint16(at + 22, true);
80
+ if (code !== 1 ||
81
+ bits !== 16 ||
82
+ (channels !== 1 && channels !== 2) ||
83
+ hz === 0 ||
84
+ hz > MOST_TAKE_HZ ||
85
+ blockAlign !== channels * 2 ||
86
+ bytesPerSecond !== hz * blockAlign) {
87
+ return null;
88
+ }
89
+ format = { hz, channels };
90
+ }
91
+ else if (chunk === 'data') {
92
+ if (format === null || data !== null || size === 0 || size % (format.channels * 2) !== 0)
93
+ return null;
94
+ data = { at: at + 8, size };
95
+ }
96
+ at += 8 + size + (size % 2);
97
+ }
98
+ if (format === null || data === null)
99
+ return null;
100
+ const { hz, channels } = format;
101
+ const { at: dataAt, size: dataSize } = data;
102
+ return {
103
+ seconds: dataSize / (hz * channels * 2),
104
+ bare() {
105
+ const out = new Uint8Array(HEADER_BYTES + dataSize);
106
+ writeWaveHeader(out.buffer, hz, channels);
107
+ out.set(new Uint8Array(bytes, dataAt, dataSize), HEADER_BYTES);
108
+ return out.buffer;
109
+ },
110
+ };
111
+ }
@@ -0,0 +1,32 @@
1
+ import type { PlaySeconds } from '@open-artifact/motion-core';
2
+ /** The span a sound is heard over, in play time; sounds come ordered by `start`. */
3
+ export interface HeardSpan {
4
+ readonly start: PlaySeconds;
5
+ readonly end: PlaySeconds;
6
+ }
7
+ /** How far ahead of the playhead sounds are handed to the audio context, in context seconds. */
8
+ export declare const LOOKAHEAD_SECONDS = 1.2;
9
+ /** How often the scheduler looks ahead. */
10
+ export declare const SCHEDULER_INTERVAL_MS = 200;
11
+ /** A start this far in the future, so the first sounds are not late. */
12
+ export declare const START_DELAY_SECONDS = 0.06;
13
+ /** The play time `contextTime` stands for, in a pass that put play time `0` at context time `origin` and plays at `rate`. */
14
+ export declare function playTimeAt(contextTime: number, origin: number, rate: number): PlaySeconds;
15
+ /** The context time play time `play` sounds at, in that same pass. */
16
+ export declare function contextTimeOf(play: number, origin: number, rate: number): number;
17
+ /** The context origin of a pass that starts playing `from` at `contextNow` + the start delay. */
18
+ export declare function passOrigin(contextNow: number, from: PlaySeconds, rate: number): number;
19
+ /** The play time up to which sounds are due: the playhead plus the lookahead, both as played at `rate`. */
20
+ export declare function lookaheadHorizon(contextNow: number, origin: number, rate: number, lookahead?: number): PlaySeconds;
21
+ /** The first sound still heard at or after `from`, counting from the start. */
22
+ export declare function firstHeard(spans: readonly HeardSpan[], from: PlaySeconds): number;
23
+ /**
24
+ * The sounds due by `horizon`, from `index` on: those that start to be heard by then, less any already over at `from`.
25
+ * `next` is where the next look starts.
26
+ */
27
+ export declare function dueSounds(spans: readonly HeardSpan[], index: number, from: PlaySeconds, horizon: PlaySeconds): {
28
+ due: number[];
29
+ next: number;
30
+ };
31
+ /** Every sound heard at or after `from`, as an offline render schedules them all at once. */
32
+ export declare function heardFrom(spans: readonly HeardSpan[], from: PlaySeconds): number[];
package/src/window.js ADDED
@@ -0,0 +1,59 @@
1
+ import { playSeconds } from '@open-artifact/motion-core';
2
+ /** How far ahead of the playhead sounds are handed to the audio context, in context seconds. */
3
+ export const LOOKAHEAD_SECONDS = 1.2;
4
+ /** How often the scheduler looks ahead. */
5
+ export const SCHEDULER_INTERVAL_MS = 200;
6
+ /** A start this far in the future, so the first sounds are not late. */
7
+ export const START_DELAY_SECONDS = 0.06;
8
+ /** The play time `contextTime` stands for, in a pass that put play time `0` at context time `origin` and plays at `rate`. */
9
+ export function playTimeAt(contextTime, origin, rate) {
10
+ return playSeconds((contextTime - origin) * rate);
11
+ }
12
+ /** The context time play time `play` sounds at, in that same pass. */
13
+ export function contextTimeOf(play, origin, rate) {
14
+ return origin + play / rate;
15
+ }
16
+ /** The context origin of a pass that starts playing `from` at `contextNow` + the start delay. */
17
+ export function passOrigin(contextNow, from, rate) {
18
+ return contextNow + START_DELAY_SECONDS - from / rate;
19
+ }
20
+ /** The play time up to which sounds are due: the playhead plus the lookahead, both as played at `rate`. */
21
+ export function lookaheadHorizon(contextNow, origin, rate, lookahead = LOOKAHEAD_SECONDS) {
22
+ return playSeconds((contextNow - origin + lookahead) * rate);
23
+ }
24
+ /** The first sound still heard at or after `from`, counting from the start. */
25
+ export function firstHeard(spans, from) {
26
+ let index = 0;
27
+ while (index < spans.length && spans[index].end <= from)
28
+ index++;
29
+ return index;
30
+ }
31
+ /**
32
+ * The sounds due by `horizon`, from `index` on: those that start to be heard by then, less any already over at `from`.
33
+ * `next` is where the next look starts.
34
+ */
35
+ export function dueSounds(spans, index, from, horizon) {
36
+ const due = [];
37
+ let next = index;
38
+ while (next < spans.length) {
39
+ const span = spans[next];
40
+ if (span.end <= from) {
41
+ next++;
42
+ continue;
43
+ }
44
+ if (span.start > horizon)
45
+ break;
46
+ due.push(next);
47
+ next++;
48
+ }
49
+ return { due, next };
50
+ }
51
+ /** Every sound heard at or after `from`, as an offline render schedules them all at once. */
52
+ export function heardFrom(spans, from) {
53
+ const heard = [];
54
+ spans.forEach((span, index) => {
55
+ if (span.end > from)
56
+ heard.push(index);
57
+ });
58
+ return heard;
59
+ }