@aeonentertainment/mixer 0.0.0-stage → 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Aeon Entertainment
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,94 @@
-# Temporary Holding Version
+# Aeon Mixer
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+A low-latency audio mixer for [Aeon Karaoke](https://github.com/aeonentertainment/aeon-karaoke).
+Phones on the LAN, wired mics and wireless mics are the microphones. The mixer takes those mic
+streams and a backing track, and plays one mix to the room.
+
+A native engine owns the audio device's output callback. TypeScript drives it through a Node
+addon, and no audio crosses into JavaScript. Mouth to speaker, the target is 20 to 30 ms. The
+package ships the addon prebuilt for macOS, Windows and Linux, so installing it needs no Rust
+toolchain. Aeon Karaoke's device service is its one consumer.
+
+## Install
+
+Each [release](https://github.com/aeonentertainment/aeon-mixer/releases) carries one tarball.
+Depend on its URL, and the lockfile pins its hash:
+
+```bash
+npm install https://github.com/aeonentertainment/aeon-mixer/releases/download/v0.2.0/aeonentertainment-mixer-0.2.0.tgz
+```
+
+## Use
+
+```ts
+import { openEngine } from "@aeonentertainment/mixer";
+
+// Opened once for the life of the process; it never reopens the device between songs
+const engine = await openEngine({ output: {} });
+
+const mixer = await engine.createMixer({
+  slots: [{ kind: "rtp" }, { kind: "l16" }, { kind: "push" }],
+  backing: {
+    stems: [
+      { kind: "instrumental", path: "song/instrumental.flac" },
+      { kind: "guideVocals", path: "song/vocals.flac" },
+    ],
+    key: -2,
+    speed: 1,
+    stemGains: { instrumental: 1, guideVocals: 0.3 },
+  },
+  rtpReturn: { host: "192.168.1.20", port: 40000, payloadType: 111, ssrc: 0x1234_5678 },
+});
+
+mixer.slots[0]?.port; // the UDP port this phone sends Opus over RTP to
+mixer.on("level", (slot, rmsDb, peakDb) => {});
+
+const { startedAt, backingOffsetMs } = await mixer.start();
+await mixer.destroy();
+await engine.close();
+```
+
+- **Slots** are the mics: 8 per engine by default, up to 16 with `openEngine({ slots })`. `rtp`
+  takes Opus over UDP from a phone or mediasoup, `l16` takes raw PCM over loopback from a house
+  mic, and `push` takes PCM the host hands to `mixer.push`. Each slot has gain, pan, mute, solo,
+  reverb, a pitch reading and a melody guide.
+- **The backing** is any format Symphonia decodes, resampled to 48 kHz. The armed key and speed
+  are rendered before `start` resolves. Live, it supports seek, pause, resume, a key change and
+  stem gains, and `start`, `seek` and `resume` report the position the lyric clock follows.
+- **`rtpReturn`** sends the mix to mediasoup's PlainTransport as 10 ms Opus with in-band FEC.
+  `host` is an IP address, never a name.
+- **Reactions** play through `engine.reactions.play` between songs and during them.
+- **The timer engine**, `openEngine({ output: { timer: true } })`, is for a box with no sound
+  card. It opens no device; its own thread renders a block at each deadline on the OS's
+  monotonic clock, and the mix leaves on `rtpReturn`. `measure-clock` prints its timing:
+  `cargo run --release -p devices --bin measure-clock` in `native/`.
+- **The offline engine**, `openEngine({ output: { offline: true } })`, opens no device and
+  renders only when its caller runs `engine.pump(blocks)`. The tests run on it.
+
+[docs/design.md](docs/design.md) section 7.2 has the full API. The reasons are in
+[docs/decisions](docs/decisions).
+
+## Development
+
+You install Node 24, pnpm, a Rust toolchain, cmake and a C++17 compiler yourself (GCC, Clang or
+the Visual Studio C++ build tools). cmake builds libopus from source, and the compiler builds the
+DSP kernels in `native/kernels`.
+
+```bash
+pnpm install                     # dependencies, including Vite+
+pnpm exec vp run build:native    # the addon the TypeScript tests load
+pnpm exec vp check               # format, lint, type-check
+pnpm exec vp test                # the TypeScript suite, on the offline engine
+pnpm exec vp pack                # build dist/
+```
+
+In `native/`, `cargo fmt --check`, `cargo clippy --all-targets -- -D warnings` and `cargo test`
+do the same for Rust. A pre-commit hook runs `vp check --fix` on staged files;
+`pnpm exec vp hooks disable` turns it off for your clone.
+
+Agents start at [AGENTS.md](AGENTS.md). Contributors start at [CONTRIBUTING.md](CONTRIBUTING.md).
+Security reports go through [SECURITY.md](SECURITY.md).
+
+## License
+
+[MIT](LICENSE)

package/dist/index.d.mts

@@ -0,0 +1,462 @@
+/**
+ * A song's stems being decoded and rendered for one mixer. JavaScript holds it from
+ * `createMixer` and installs it when the events thread says the job is done.
+ */
+declare class BackingRender {
+  /** The id the events thread's `renderProgress` and `renderFinished` events carry. */
+  get id(): number;
+}
+declare class Engine$1 {
+  /**
+   * Starts the clock on the silent graph and the events thread, which calls `on_events` with
+   * every batch it drains.
+   */
+  constructor(options: OpenOptions, onEvents: (arg: Array<NativeEvent>) => void);
+  /**
+   * Builds a mixer with one slot per kind, in order: `"rtp"`, `"l16"` or `"push"`. The rtp
+   * and l16 slots bind their ports here and report them through `Mixer.port`; a return stream
+   * gets its socket and send task here too.
+   */
+  createMixer(kinds: Array<string>, rtpReturn?: RtpReturnInput | undefined | null): Mixer$1;
+  get output(): OutputInfo$1;
+  /** The outputs this OS offers, the default marked. Empty on the offline and timer clocks. */
+  outputs(): Array<OutputEntry>;
+  /** Offline clock only: renders `blocks` blocks and returns them as interleaved stereo. */
+  pump(blocks: number): Float32Array;
+  /** Writes one slot of the snapshot and publishes the whole of it. */
+  setSlotParams(index: number, params: SlotParamsInput): void;
+  /**
+   * Moves `mixer`'s graph to the audio thread and returns the swap's id. The graph it replaces
+   * comes back through the retired ring as a `retired` event.
+   */
+  swap(mixer: Mixer$1): number;
+  /** Swaps the silent graph in: the end of a mixer. */
+  swapSilent(): number;
+  start(): number;
+  /**
+   * Starts decoding and rendering a song's stems on the worker thread. The events thread
+   * reports the job's progress under the id the result carries.
+   */
+  startRender(input: BackingInput): BackingRender;
+  seek(toFrame: number): number;
+  pause(): number;
+  resume(): number;
+  setKey(semitones: number): number;
+  stopAt(atFrame: number, fadeFrames: number): number;
+  /** Writes the stem gains of the snapshot and publishes the whole of it. */
+  setStemGains(gains: StemGainsInput): void;
+  /** Gives the live graph's slot a guide, or takes it away with nothing. No ack. */
+  setGuide(slot: number, guide?: GuideInput | undefined | null): void;
+  /** The live graph's slot forgets the stream it was hearing. No ack. */
+  resetSlot(slot: number): void;
+  /** Plays interleaved stereo on a free reaction voice; false when four already play. */
+  playReaction(stereo: Float32Array): boolean;
+  /**
+   * Stops the clock, then the network thread with every port still on it, then the events
+   * thread. Whatever graph is live is dropped here.
+   */
+  close(): void;
+}
+declare class Mixer$1 {
+  /** Moves a finished render's stems into the graph as its backing path, before `swap`. */
+  installBacking(render: BackingRender): void;
+  /** The UDP port a slot listens on, or null for a push slot. */
+  port(index: number): number | null;
+  /**
+   * Hands interleaved stereo PCM to a push slot, which folds it to mono as every slot is
+   * (docs/design.md 8.4), and returns how many frames it took.
+   */
+  push(index: number, stereo: Float32Array): number;
+  /**
+   * Records what reaches an `rtp` or `l16` slot's port to a new pcap file at `path`, or stops
+   * recording it for null.
+   */
+  recordSlot(index: number, path?: string | undefined | null): void;
+  /** Every slot's ingest counters, as the audio thread last published them. */
+  stats(): Array<SlotStatsInfo>;
+  /** The return stream's counters, or null for a mixer without one. */
+  returnStats(): RtpReturnStats | null;
+  /**
+   * The end of the mixer on this thread: closes the slots' ports, drops the push feeds, stops
+   * the return stream, and drops a graph that never started.
+   */
+  release(): void;
+}
+interface BackingInput {
+  instrumental: string;
+  guide?: string;
+  /** The armed key, rendered before start. */
+  semitones: number;
+  /** The armed speed factor, rendered before start. */
+  speed: number;
+}
+interface CorrectorInput {
+  enabled: boolean;
+  strength: number;
+  retuneMs: number;
+}
+interface GuideInput {
+  notes: Array<NoteInput>;
+  /** The key's pitch classes as bits, C in bit 0. */
+  keyMask: number;
+}
+type NativeEvent = {
+  kind: 'ack';
+  id: number;
+  positionFrames: number;
+} | {
+  kind: 'level';
+  slot: number;
+  rmsDb: number;
+  peakDb: number;
+} | {
+  kind: 'masterLevel';
+  rmsDb: number;
+  peakDb: number;
+} | {
+  kind: 'pitch';
+  slot: number;
+  positionMs: number;
+  hz: number;
+  confidence: number;
+} | {
+  kind: 'position';
+  frames: number;
+} | {
+  kind: 'backingEnded';
+} | {
+  kind: 'underrun';
+} | {
+  kind: 'fault';
+} | {
+  kind: 'retired';
+} | {
+  kind: 'renderProgress';
+  job: number;
+  fraction: number;
+} | {
+  kind: 'renderFinished';
+  job: number;
+};
+interface NoteInput {
+  midi: number;
+  startFrame: number;
+  endFrame: number;
+}
+interface OpenOptions {
+  clock: 'offline' | 'timer' | 'device';
+  /** A device clock's output: an id from `outputs`, or the default output. */
+  device?: string;
+  periodFrames: number;
+  /** The jitter buffer's playout delay for every `rtp` and `l16` slot. */
+  playoutDelayFrames: number;
+}
+interface OutputEntry {
+  id: string;
+  name: string;
+  isDefault: boolean;
+}
+interface OutputInfo$1 {
+  sampleRate: number;
+  periodFrames: number;
+  latencyMs: number;
+}
+interface ReverbInput {
+  enabled: boolean;
+  roomSize: number;
+  mix: number;
+}
+/** `MixerConfig.rtpReturn`, range-checked in `src/` before it crosses. */
+interface RtpReturnInput {
+  /** An IP literal: a name would be resolved on the control thread. */
+  host: string;
+  port: number;
+  payloadType: number;
+  ssrc: number;
+}
+/** What `Mixer.stats` says about the return stream. */
+interface RtpReturnStats {
+  /** Blocks the audio thread dropped because the network thread had not drained the ring. */
+  droppedBlocks: number;
+}
+interface SlotParamsInput {
+  /** Linear amplitude. */
+  gain: number;
+  /** -1 left to 1 right. */
+  pan: number;
+  mute: boolean;
+  solo: boolean;
+  corrector: CorrectorInput;
+  reverb: ReverbInput;
+  guideOffsetMs: number;
+}
+/** `SlotStats` as JavaScript reads it. */
+interface SlotStatsInfo {
+  pushed: number;
+  lost: number;
+  late: number;
+  concealed: number;
+  resyncs: number;
+  jitterMs: number;
+}
+interface StemGainsInput {
+  instrumental: number;
+  guide: number;
+}
+declare const NativeEngine: typeof Engine$1;
+type NativeEngine = Engine$1;
+type NativeMixer = Mixer$1;
+type Capability = "guideVocal" | "key" | "speed" | "seek" | "shortVersion" | "pause" | "level";
+type IngestKind = "rtp" | "l16" | "push";
+type StemKind = "instrumental" | "guideVocals";
+interface OpenEngineOptions {
+  /**
+   * A device; `timer` for a box with no sound card, whose own thread renders on the OS's clock
+   * and whose mix leaves only on a mixer's `rtpReturn`; `offline` for tests, which pump it.
+   */
+  output: {
+    device?: string;
+    periodFrames?: 64 | 128;
+  } | {
+    timer: true;
+  } | {
+    offline: true;
+  };
+  /** Fixed for the engine's life, default 8, at most 16. */
+  slots?: number;
+  /** The ingest jitter buffer, default 15. */
+  playoutDelayMs?: number;
+}
+interface OutputInfo {
+  readonly sampleRate: 48000;
+  readonly periodFrames: number;
+  readonly latencyMs: number;
+}
+interface MixerConfig {
+  slots: ReadonlyArray<{
+    kind: IngestKind;
+  }>;
+  backing?: {
+    stems: ReadonlyArray<{
+      kind: StemKind;
+      path: string;
+    }>;
+    /** Semitones, rendered before start. */
+    key: number;
+    /** A factor, rendered before start. */
+    speed: number;
+    stemGains: Record<StemKind, number>;
+    stopAtMs?: number;
+  };
+  rtpReturn?: {
+    host: string;
+    port: number;
+    payloadType: number;
+    ssrc: number;
+  };
+}
+interface SlotSettings {
+  /** Linear amplitude, 0 to 4. */
+  gain: number;
+  /** -1 left to 1 right. */
+  pan: number;
+  mute: boolean;
+  solo: boolean;
+}
+type SlotEffect = {
+  kind: "pitch-correction";
+  enabled: boolean;
+  strength: number;
+  retuneMs: number;
+} | {
+  kind: "reverb";
+  enabled: boolean;
+  roomSize: number;
+  mix: number;
+};
+interface MelodyGuide {
+  notes: ReadonlyArray<{
+    midi: number;
+    startMs: number;
+    endMs: number;
+  }>;
+  /** The key's pitch classes as bits, C in bit 0. */
+  keyMask: number;
+}
+interface SlotStats {
+  pushed: number;
+  lost: number;
+  late: number;
+  concealed: number;
+  resyncs: number;
+  jitterMs: number;
+}
+type EngineEvents = {
+  fault: (error: Error) => void;
+  /** The mix after the clipper, reactions included, between songs too. */
+  masterLevel: (rmsDb: number, peakDb: number) => void;
+};
+type MixerEvents = {
+  level: (slot: number, rmsDb: number, peakDb: number) => void;
+  pitch: (slot: number, positionMs: number, hz: number, confidence: number) => void;
+  position: (positionMs: number) => void;
+  renderProgress: (fraction: number) => void;
+  backingEnded: () => void;
+  error: (error: Error) => void;
+};
+type BackingConfig = NonNullable<MixerConfig["backing"]>;
+/** What the backing path needs from its engine: word of its render job, as the events thread reports it. */
+interface RenderHost {
+  readonly native: NativeEngine;
+  /**
+   * Reports the render job's progress until it finishes, then resolves; rejects when the engine
+   * stops first.
+   */
+  watchRender(id: number, onProgress: (fraction: number) => void): Promise<void>;
+}
+type Handler = (...args: never[]) => void;
+/** Typed listeners for one object's events. `on` returns the unsubscribe. */
+declare class Emitter<Events extends Record<keyof Events, Handler>> {
+  #private;
+  on<K extends keyof Events>(event: K, listener: Events[K]): () => void;
+  emit<K extends keyof Events>(event: K, ...args: Parameters<Events[K]>): void;
+}
+/** What a mixer needs from its engine. Internal to the package. */
+interface MixerHost extends RenderHost {
+  readonly output: OutputInfo;
+  /**
+   * Sends a command carrying an id and resolves on its ack with the backing position. `onTimeout`
+   * hears the rejection when no ack comes in time.
+   */
+  command(send: () => number, onTimeout: (error: Error) => void): Promise<number>;
+  /**
+   * Swaps `graph` in as the live graph, routing its events to `events`. Rejects while another
+   * mixer is live. `onTimeout` is `command`'s.
+   */
+  swapIn(mixer: Mixer, graph: NativeMixer, events: Emitter<MixerEvents>, onTimeout: (error: Error) => void): Promise<number>;
+  /**
+   * Swaps the silent graph back in; resolves once the mixer's graph is back and dropped, or at
+   * once on a faulted engine, which hands nothing back.
+   */
+  swapOut(mixer: Mixer): Promise<void>;
+  /** Hears the engine's fault until the returned function is called. */
+  onFault(listener: (error: Error) => void): () => void;
+  isLive(mixer: Mixer): boolean;
+  anyLive(): boolean;
+  underruns(): number;
+}
+/**
+ * One mixing graph for one performance. Its `rtp` and `l16` slots listen on their ports from
+ * here until `destroy`; its return stream sends the mix from `start` until `destroy`. Settings
+ * made while armed are kept here and written to the engine when `start` swaps the graph in, so
+ * arming the next song never touches the live one.
+ */
+export declare class Mixer {
+  #private;
+  /** Each slot's kind and, for `rtp` and `l16`, the UDP port it listens on; `null` for `push`. */
+  readonly slots: ReadonlyArray<{
+    index: number;
+    kind: IngestKind;
+    port: number | null;
+  }>;
+  constructor(host: MixerHost, kinds: ReadonlyArray<IngestKind>, events: Emitter<MixerEvents>, backing: BackingConfig | null, rtpReturn: RtpReturnInput | null);
+  setSlot(index: number, settings: Partial<SlotSettings>): void;
+  /** The slot's whole chain: an effect not in `effects` is disabled. */
+  setSlotEffects(index: number, effects: ReadonlyArray<SlotEffect>): void;
+  setSlotGuide(index: number, guide: MelodyGuide | null, guideOffsetMs?: number): void;
+  /**
+   * Forget the stream a slot was hearing; the next packets, or the next pushed audio, are a new
+   * singer. An armed mixer has heard nothing yet, so there is nothing to forget.
+   */
+  resetSlot(index: number): void;
+  /**
+   * Writes every datagram that reaches an `rtp` or `l16` slot's port to a new pcap file at `path`,
+   * with the moment the network thread read it, until `null`, another path or `destroy`. The file
+   * is created here and never over an existing one, so a bad path throws. A recording holds a
+   * singer's voice and their phone's LAN address: it is made only when a person asks for one, and
+   * nothing in this package sends it anywhere. Start it after `start`, so a replay begins where
+   * the live slot did. docs/design.md 8.4.
+   */
+  recordSlot(index: number, path: string | null): void;
+  /**
+   * Hands interleaved stereo PCM to a push slot, folded to mono as every slot is. Returns the
+   * frames taken; the rest did not fit and is dropped, since late audio is worse than lost audio.
+   * Throws on a slot of another kind.
+   */
+  push(index: number, stereo: Float32Array): number;
+  /**
+   * Waits for the backing's render and swaps the graph in; a render that failed rejects with the
+   * message `error` carried. With the render already done, the swap is sent before this returns
+   * to the caller. Nothing is written to the engine while another mixer is live, so a refused
+   * start leaves the song playing untouched.
+   */
+  start(): Promise<{
+    startedAt: number;
+    backingOffsetMs: number;
+    speedFactor: number;
+  }>;
+  /** Resolves with the position the seek landed on, once its fade out is done. */
+  seek(toMs: number): Promise<{
+    positionMs: number;
+  }>;
+  /** Resolves once the fade out is done and the playhead holds. */
+  pause(): Promise<void>;
+  resume(): Promise<{
+    positionMs: number;
+  }>;
+  /** The live key, on top of the armed one; its crossfade follows the ack. */
+  setBackingKey(semitones: number): Promise<void>;
+  /** A parameter write, held while armed like the slot settings. */
+  setStemGain(kind: StemKind, gain: number): void;
+  on<K extends keyof MixerEvents>(event: K, listener: MixerEvents[K]): () => void;
+  /**
+   * Each slot's ingest counters as the audio thread last published them, the engine's underruns,
+   * and the blocks the return stream dropped, `null` without one.
+   */
+  stats(): {
+    slots: ReadonlyArray<SlotStats>;
+    underruns: number;
+    rtpReturn: {
+      droppedBlocks: number;
+    } | null;
+  };
+  /**
+   * Closes the slots' ports, stops the return stream, swaps the silent graph in when this one is
+   * live, and resolves once this graph is back and dropped.
+   */
+  destroy(): Promise<void>;
+}
+/** An engine on the test clock. Its caller pumps it; nothing plays. */
+type OfflineEngine = Engine & Required<Pick<Engine, "pump">>;
+/**
+ * Opens the engine for the life of the process: the device, the timer clock on a box with no
+ * sound card, or the offline driver, running the silent graph until a mixer starts.
+ */
+export declare function openEngine(options: OpenEngineOptions & {
+  output: {
+    offline: true;
+  };
+}): Promise<OfflineEngine>;
+export declare function openEngine(options: OpenEngineOptions): Promise<Engine>;
+export declare class Engine {
+  #private;
+  readonly capabilities: ReadonlySet<Capability>;
+  readonly output: OutputInfo;
+  /** Offline engines only: render blocks and return them as interleaved stereo. */
+  readonly pump?: (blocks: number) => Float32Array;
+  constructor(options: OpenEngineOptions);
+  /** Room-lifetime reaction voices, four at once. False when a fifth is dropped while four play. */
+  readonly reactions: {
+    play: (samples: Float32Array, channels: 1 | 2) => boolean;
+  };
+  outputDevices(): Promise<ReadonlyArray<{
+    id: string;
+    name: string;
+    isDefault: boolean;
+  }>>;
+  createMixer(config: MixerConfig): Promise<Mixer>;
+  on<K extends keyof EngineEvents>(event: K, listener: EngineEvents[K]): () => void;
+  /** Destroys the live mixer, stops the clock and joins the events thread. */
+  close(): Promise<void>;
+}
+export type { Capability, EngineEvents, IngestKind, MelodyGuide, MixerConfig, MixerEvents, OfflineEngine, OpenEngineOptions, OutputInfo, SlotEffect, SlotSettings, SlotStats, StemKind };