@aeonentertainment/mixer 0.3.0 → 0.5.0

package/README.md

@@ -15,7 +15,7 @@ Each [release](https://github.com/aeonentertainment/aeon-mixer/releases) carries
 Depend on its URL, and the lockfile pins its hash:
 
 ```bash
-npm install https://github.com/aeonentertainment/aeon-mixer/releases/download/v0.3.0/aeonentertainment-mixer-0.3.0.tgz
+npm install https://github.com/aeonentertainment/aeon-mixer/releases/download/v0.5.0/aeonentertainment-mixer-0.5.0.tgz
 ```
 
 ## Use
@@ -52,7 +52,9 @@ await engine.close();
   takes Opus over UDP from a phone or mediasoup, `l16` takes raw PCM over loopback from a house
   mic, and `push` takes PCM the host hands to `mixer.push`. Each slot has gain, pan, mute, solo,
   reverb, a pitch reading, a melody guide and pitch correction toward it, whose delay `stats()`
-  reports.
+  reports. A slot can also take the room's echo of the mix out of its input and duck itself when
+  its loop through the speakers starts to howl; both add no delay, and `stats()` reports what
+  they found.
 - **The backing** is any format Symphonia decodes, resampled to 48 kHz. The armed key and speed
   are rendered before `start` resolves. Live, it supports seek, pause, resume, a key change and
   stem gains, and `start`, `seek` and `resume` report the position the lyric clock follows.
@@ -63,6 +65,11 @@ await engine.close();
   card. It opens no device; its own thread renders a block at each deadline on the OS's
   monotonic clock, and the mix leaves on `rtpReturn`. `measure-clock` prints its timing:
   `cargo run --release -p devices --bin measure-clock` in `native/`.
+- **The output** is the OS default, unless `openEngine({ output: { device } })` pins one by an id
+  from `engine.outputDevices()`. When the default changes, or the output goes away, the song
+  moves to the new output after a short gap, a pinned engine moving back when its output
+  returns. `engine.output` names the output playing and its latency, and `outputChanged` fires
+  after each move with both and the gap.
 - **The offline engine**, `openEngine({ output: { offline: true } })`, opens no device and
   renders only when its caller runs `engine.pump(blocks)`. The tests run on it.
 

package/dist/index.d.mts

@@ -18,6 +18,7 @@ declare class Engine$1 {
    * gets its socket and send task here too.
    */
   createMixer(kinds: Array<string>, rtpReturn?: RtpReturnInput | undefined | null): Mixer$1;
+  /** The output playing now. After a move the events thread sends the same as `outputChanged`. */
   get output(): OutputInfo$1;
   /** The outputs this OS offers, the default marked. Empty on the offline and timer clocks. */
   outputs(): Array<OutputEntry>;
@@ -72,7 +73,10 @@ declare class Mixer$1 {
    * recording it for null.
    */
   recordSlot(index: number, path?: string | undefined | null): void;
-  /** Every slot's ingest counters and corrector delay, as the audio thread last published them. */
+  /**
+   * Every slot's ingest counters, corrector delay and echo findings, as the audio thread last
+   * published them.
+   */
   stats(): Array<SlotStatsInfo>;
   /** The return stream's counters, or null for a mixer without one. */
   returnStats(): RtpReturnStats | null;
@@ -90,11 +94,23 @@ interface BackingInput {
   /** The armed speed factor, rendered before start. */
   speed: number;
 }
+interface CleanupInput {
+  enabled: boolean;
+}
 interface CorrectorInput {
   enabled: boolean;
   strength: number;
   retuneMs: number;
 }
+/** What a slot's echo canceller has found, as JavaScript reads it. */
+interface EchoInfo {
+  /** From the master being rendered to its echo reaching the slot. */
+  pathMs: number;
+  /** How far under the master the echo arrives, over the last second. */
+  bleedDb: number;
+  /** How much quieter the slot came out than it went in, over the last second. */
+  removedDb: number;
+}
 interface GuideInput {
   notes: Array<NoteInput>;
   /** The key's pitch classes as bits, C in bit 0. */
@@ -119,6 +135,8 @@ type NativeEvent = {
   positionMs: number;
   hz: number;
   confidence: number;
+  targetMidi?: number;
+  errorSemitones: number;
 } | {
   kind: 'position';
   frames: number;
@@ -137,6 +155,10 @@ type NativeEvent = {
 } | {
   kind: 'renderFinished';
   job: number;
+} | {
+  kind: 'outputChanged';
+  output: OutputInfo$1;
+  gapMs: number;
 };
 interface NoteInput {
   midi: number;
@@ -145,11 +167,18 @@ interface NoteInput {
 }
 interface OpenOptions {
   clock: 'offline' | 'timer' | 'device';
-  /** A device clock's output: an id from `outputs`, or the default output. */
+  /** A device clock's output: an id from `outputs` to pin, or none to follow the default output. */
   device?: string;
   periodFrames: number;
-  /** The jitter buffer's playout delay for every `rtp` and `l16` slot. */
-  playoutDelayFrames: number;
+  /**
+   * Pins the jitter buffer's playout delay for every `rtp` and `l16` slot. Without it, each
+   * stream chooses its own.
+   */
+  playoutDelayFrames?: number;
+}
+interface OutputDevice {
+  id: string;
+  name: string;
 }
 interface OutputEntry {
   id: string;
@@ -160,6 +189,8 @@ interface OutputInfo$1 {
   sampleRate: number;
   periodFrames: number;
   latencyMs: number;
+  /** The output playing now; none on the offline clock. */
+  device?: OutputDevice;
 }
 interface ReverbInput {
   enabled: boolean;
@@ -188,6 +219,10 @@ interface SlotParamsInput {
   solo: boolean;
   corrector: CorrectorInput;
   reverb: ReverbInput;
+  voice: VoiceInput;
+  echoCancel: SwitchInput;
+  cleanup: CleanupInput;
+  howlGuard: SwitchInput;
   guideOffsetMs: number;
 }
 /** `SlotStats` as JavaScript reads it. */
@@ -195,16 +230,29 @@ interface SlotStatsInfo {
   pushed: number;
   lost: number;
   late: number;
+  lateAtOpening: number;
   concealed: number;
   resyncs: number;
   jitterMs: number;
   correctorDelayMs: number;
   correctorMaxDelayMs: number;
+  /** Null while the canceller is off or has yet to find and learn the room's echo. */
+  echo: EchoInfo | null;
+  playoutDelayMs: number;
+  howlCatches: number;
 }
 interface StemGainsInput {
   instrumental: number;
   guide: number;
 }
+/** An effect with no knob but on and off. */
+interface SwitchInput {
+  enabled: boolean;
+}
+interface VoiceInput {
+  enabled: boolean;
+  formantSemitones: number;
+}
 declare const NativeEngine: typeof Engine$1;
 type NativeEngine = Engine$1;
 type NativeMixer = Mixer$1;
@@ -213,8 +261,9 @@ type IngestKind = "rtp" | "l16" | "push";
 type StemKind = "instrumental" | "guideVocals";
 interface OpenEngineOptions {
   /**
-   * A device; `timer` for a box with no sound card, whose own thread renders on the OS's clock
-   * and whose mix leaves only on a mixer's `rtpReturn`; `offline` for tests, which pump it.
+   * A device, pinned by `device` or, left out, following the OS default; `timer` for a box with
+   * no sound card, whose own thread renders on the OS's clock and whose mix leaves only on a
+   * mixer's `rtpReturn`; `offline` for tests, which pump it.
    */
   output: {
     device?: string;
@@ -226,13 +275,21 @@ interface OpenEngineOptions {
   };
   /** Fixed for the engine's life, default 8, at most 16. */
   slots?: number;
-  /** The ingest jitter buffer, default 15. */
+  /**
+   * Pins the ingest jitter buffer to this delay. Leave it out and each slot chooses its own from
+   * what its stream shows, opening at 15; `stats()` reports where each one is.
+   */
   playoutDelayMs?: number;
 }
 interface OutputInfo {
   readonly sampleRate: 48000;
   readonly periodFrames: number;
   readonly latencyMs: number;
+  /** The output playing now, as `outputDevices()` lists it; null on the offline clock. */
+  readonly device: {
+    readonly id: string;
+    readonly name: string;
+  } | null;
 }
 interface MixerConfig {
   slots: ReadonlyArray<{
@@ -275,6 +332,33 @@ type SlotEffect = {
   enabled: boolean;
   roomSize: number;
   mix: number;
+} |
+/**
+ * A formant move in semitones from the singer's own, -12 to 12, with no change of pitch and no
+ * added delay. Decision 0014 has the first voices' settings.
+ */
+{
+  kind: "voice";
+  enabled: boolean;
+  formantSemitones: number;
+} |
+/** Takes the room's echo of the mix out of the slot's input, with no added delay. */
+{
+  kind: "echo-cancel";
+  enabled: boolean;
+} |
+/**
+ * A high-pass and gate ahead of the corrector and a compressor after the meter. None reads
+ * ahead or buffers; decision 0012 has their numbers and costs.
+ */
+{
+  kind: "cleanup";
+  enabled: boolean;
+} |
+/** Ducks the slot when its loop through the speakers starts to howl, with no added delay. */
+{
+  kind: "howl-guard";
+  enabled: boolean;
 };
 interface MelodyGuide {
   notes: ReadonlyArray<{
@@ -289,6 +373,8 @@ interface SlotStats {
   pushed: number;
   lost: number;
   late: number;
+  /** Of `late`, the packets that would also have been late had the delay stayed where it opened. */
+  lateAtOpening: number;
   concealed: number;
   resyncs: number;
   jitterMs: number;
@@ -299,15 +385,39 @@ interface SlotStats {
   correctorDelayMs: number;
   /** The most that delay can reach; 0 with correction off. It holds still, so a budget adds this. */
   correctorMaxDelayMs: number;
+  /**
+   * What the echo canceller found: `pathMs` from the mix being rendered to its echo reaching the
+   * slot, `bleedDb` how far under the mix that echo arrives, `removedDb` how much quieter the slot
+   * came out than it went in, both over the last second. Null while it is off, or until it has
+   * found the echo and learned it.
+   */
+  echo: {
+    pathMs: number;
+    bleedDb: number;
+    removedDb: number;
+  } | null;
+  /** The delay the stream's earliest packets wait now; zero for a `push` slot. */
+  playoutDelayMs: number;
+  /** How often the howl guard has ducked this slot. */
+  howlCatches: number;
 }
 type EngineEvents = {
   fault: (error: Error) => void;
+  /** After every move to another output; `gapMs` is how long the room heard nothing. */
+  outputChanged: (output: OutputInfo, gapMs: number) => void;
   /** The mix after the clipper, reactions included, between songs too. */
   masterLevel: (rmsDb: number, peakDb: number) => void;
 };
 type MixerEvents = {
   level: (slot: number, rmsDb: number, peakDb: number) => void;
-  pitch: (slot: number, positionMs: number, hz: number, confidence: number) => void;
+  /**
+   * Every 10 ms, on the guide's clock. `targetMidi` is the guide note the corrector pulls toward,
+   * after the backing's key shift: null with correction off, with no guide, or between notes,
+   * where it pulls toward the key. `errorSemitones` is the class it pulls toward less the
+   * reading, in the singer's own octave, so positive is flat: 0 with correction off or while
+   * unvoiced.
+   */
+  pitch: (slot: number, positionMs: number, hz: number, confidence: number, targetMidi: number | null, errorSemitones: number) => void;
   position: (positionMs: number) => void;
   renderProgress: (fraction: number) => void;
   backingEnded: () => void;
@@ -332,7 +442,8 @@ declare class Emitter<Events extends Record<keyof Events, Handler>> {
 }
 /** What a mixer needs from its engine. Internal to the package. */
 interface MixerHost extends RenderHost {
-  readonly output: OutputInfo;
+  /** The output playing now; it changes when the engine moves. */
+  output(): OutputInfo;
   /**
    * Sends a command carrying an id and resolves on its ack with the backing position. `onTimeout`
    * hears the rejection when no ack comes in time.
@@ -450,10 +561,14 @@ export declare function openEngine(options: OpenEngineOptions): Promise<Engine>;
 export declare class Engine {
   #private;
   readonly capabilities: ReadonlySet<Capability>;
-  readonly output: OutputInfo;
   /** Offline engines only: render blocks and return them as interleaved stereo. */
   readonly pump?: (blocks: number) => Float32Array;
   constructor(options: OpenEngineOptions);
+  /**
+   * The output playing now. An engine opened with no `output.device` follows the OS default, and
+   * one with a `device` plays there while it exists; `outputChanged` fires after every move.
+   */
+  get output(): OutputInfo;
   /** Room-lifetime reaction voices, four at once. False when a fifth is dropped while four play. */
   readonly reactions: {
     play: (samples: Float32Array, channels: 1 | 2) => boolean;

package/dist/index.mjs

@@ -230,6 +230,13 @@ function defaultSlotParams() {
 			roomSize: 0,
 			mix: 0
 		},
+		voice: {
+			enabled: false,
+			formantSemitones: 0
+		},
+		echoCancel: { enabled: false },
+		cleanup: { enabled: false },
+		howlGuard: { enabled: false },
 		guideOffsetMs: 0
 	};
 }
@@ -305,11 +312,26 @@ var Mixer = class {
 					retuneMs: finiteIn("retuneMs", effect.retuneMs, 0, 1e3)
 				};
 				break;
-			case "reverb": next.reverb = {
-				enabled: effect.enabled,
-				roomSize: finiteIn("roomSize", effect.roomSize, 0, 1),
-				mix: finiteIn("mix", effect.mix, 0, 1)
-			};
+			case "reverb":
+				next.reverb = {
+					enabled: effect.enabled,
+					roomSize: finiteIn("roomSize", effect.roomSize, 0, 1),
+					mix: finiteIn("mix", effect.mix, 0, 1)
+				};
+				break;
+			case "voice":
+				next.voice = {
+					enabled: effect.enabled,
+					formantSemitones: finiteIn("formantSemitones", effect.formantSemitones, -12, 12)
+				};
+				break;
+			case "echo-cancel":
+				next.echoCancel = { enabled: effect.enabled };
+				break;
+			case "cleanup":
+				next.cleanup = { enabled: effect.enabled };
+				break;
+			case "howl-guard": next.howlGuard = { enabled: effect.enabled };
 		}
 		this.#write(slot, next);
 	}
@@ -393,7 +415,7 @@ var Mixer = class {
 			started,
 			stopping
 		]);
-		const { latencyMs } = this.#host.output;
+		const { latencyMs } = this.#host.output();
 		return {
 			startedAt: Date.now() + latencyMs,
 			backingOffsetMs: latencyMs,
@@ -488,10 +510,14 @@ var Mixer = class {
 	}
 };
 function defaultEffects() {
-	const { corrector, reverb } = defaultSlotParams();
+	const { corrector, reverb, voice, cleanup, echoCancel, howlGuard } = defaultSlotParams();
 	return {
 		corrector,
-		reverb
+		reverb,
+		voice,
+		cleanup,
+		echoCancel,
+		howlGuard
 	};
 }
 function toGuideInput(guide) {
@@ -522,7 +548,7 @@ var Engine = class {
 		"pause",
 		"level"
 	]);
-	output;
+	#output;
 	/** How many slots a mixer may have, fixed at open. */
 	#slots;
 	/** Offline engines only: render blocks and return them as interleaved stereo. */
@@ -540,7 +566,7 @@ var Engine = class {
 	#state = "running";
 	constructor(options) {
 		this.#slots = integerIn("slots", options.slots ?? 8, 1, 16);
-		const playoutDelayFrames = msToFrames(finiteIn("playoutDelayMs", options.playoutDelayMs ?? 15, 5, 100));
+		const pinned = options.playoutDelayMs === void 0 ? {} : { playoutDelayFrames: msToFrames(finiteIn("playoutDelayMs", options.playoutDelayMs, 5, 100)) };
 		const deliver = (batch) => {
 			this.#deliver(batch);
 		};
@@ -548,13 +574,13 @@ var Engine = class {
 			this.#native = new NativeEngine({
 				clock: "offline",
 				periodFrames: 128,
-				playoutDelayFrames
+				...pinned
 			}, deliver);
 			this.pump = (blocks) => this.#native.pump(integerIn("blocks", blocks, 1, 1 << 20));
 		} else if ("timer" in options.output) this.#native = new NativeEngine({
 			clock: "timer",
 			periodFrames: 128,
-			playoutDelayFrames
+			...pinned
 		}, deliver);
 		else {
 			const { device, periodFrames = 128 } = options.output;
@@ -562,14 +588,14 @@ var Engine = class {
 			this.#native = new NativeEngine({
 				clock: "device",
 				periodFrames,
-				playoutDelayFrames,
+				...pinned,
 				...device === void 0 ? {} : { device }
 			}, deliver);
 		}
-		this.output = outputInfo(this.#native.output);
+		this.#output = outputInfo(this.#native.output);
 		this.#host = {
 			native: this.#native,
-			output: this.output,
+			output: () => this.#output,
 			command: (send, onTimeout) => this.#command(send, onTimeout),
 			watchRender: (id, onProgress) => new Promise((resolve, reject) => {
 				this.#renders.set(id, {
@@ -586,6 +612,13 @@ var Engine = class {
 			underruns: () => this.#underruns
 		};
 	}
+	/**
+	* The output playing now. An engine opened with no `output.device` follows the OS default, and
+	* one with a `device` plays there while it exists; `outputChanged` fires after every move.
+	*/
+	get output() {
+		return this.#output;
+	}
 	/** Room-lifetime reaction voices, four at once. False when a fifth is dropped while four play. */
 	reactions = { play: (samples, channels) => {
 		if (channels !== 1 && channels !== 2) throw new RangeError(`channels must be 1 or 2, got ${String(channels)}`);
@@ -654,7 +687,7 @@ var Engine = class {
 				this.#live?.events.emit("level", event.slot, event.rmsDb, event.peakDb);
 				break;
 			case "pitch":
-				this.#live?.events.emit("pitch", event.slot, event.positionMs, event.hz, event.confidence);
+				this.#live?.events.emit("pitch", event.slot, event.positionMs, event.hz, event.confidence, event.targetMidi ?? null, event.errorSemitones);
 				break;
 			case "position":
 				this.#live?.events.emit("position", framesToMs(event.frames));
@@ -677,6 +710,10 @@ var Engine = class {
 			case "renderProgress":
 				this.#renders.get(event.job)?.onProgress(event.fraction);
 				break;
+			case "outputChanged":
+				this.#output = outputInfo(event.output);
+				this.#events.emit("outputChanged", this.#output, event.gapMs);
+				break;
 			case "renderFinished": {
 				const render = this.#renders.get(event.job);
 				this.#renders.delete(event.job);
@@ -705,10 +742,15 @@ var Engine = class {
 };
 function outputInfo(info) {
 	if (info.sampleRate !== 48e3) throw new Error(`the engine runs at 48 kHz; the addon reports ${info.sampleRate}`);
+	const { periodFrames, latencyMs, device } = info;
 	return {
 		sampleRate: 48e3,
-		periodFrames: info.periodFrames,
-		latencyMs: info.latencyMs
+		periodFrames,
+		latencyMs,
+		device: device === void 0 ? null : {
+			id: device.id,
+			name: device.name
+		}
 	};
 }
 function monoToStereo(mono) {

package/native/binding/index.d.ts

@@ -30,6 +30,7 @@ export declare class Engine {
    * gets its socket and send task here too.
    */
   createMixer(kinds: Array<string>, rtpReturn?: RtpReturnInput | undefined | null): Mixer
+  /** The output playing now. After a move the events thread sends the same as `outputChanged`. */
   get output(): OutputInfo
   /** The outputs this OS offers, the default marked. Empty on the offline and timer clocks. */
   outputs(): Array<OutputEntry>
@@ -85,7 +86,10 @@ export declare class Mixer {
    * recording it for null.
    */
   recordSlot(index: number, path?: string | undefined | null): void
-  /** Every slot's ingest counters and corrector delay, as the audio thread last published them. */
+  /**
+   * Every slot's ingest counters, corrector delay and echo findings, as the audio thread last
+   * published them.
+   */
   stats(): Array<SlotStatsInfo>
   /** The return stream's counters, or null for a mixer without one. */
   returnStats(): RtpReturnStats | null
@@ -105,6 +109,10 @@ export interface BackingInput {
   speed: number
 }
 
+export interface CleanupInput {
+  enabled: boolean
+}
+
 /** What drives the audio thread. */
 export declare const enum ClockKind {
   /** The offline driver, pumped from JavaScript. */
@@ -121,6 +129,16 @@ export interface CorrectorInput {
   retuneMs: number
 }
 
+/** What a slot's echo canceller has found, as JavaScript reads it. */
+export interface EchoInfo {
+  /** From the master being rendered to its echo reaching the slot. */
+  pathMs: number
+  /** How far under the master the echo arrives, over the last second. */
+  bleedDb: number
+  /** How much quieter the slot came out than it went in, over the last second. */
+  removedDb: number
+}
+
 export interface GuideInput {
   notes: Array<NoteInput>
   /** The key's pitch classes as bits, C in bit 0. */
@@ -131,7 +149,7 @@ export type NativeEvent =
   | { kind: 'ack'; id: number; positionFrames: number }
   | { kind: 'level'; slot: number; rmsDb: number; peakDb: number }
   | { kind: 'masterLevel'; rmsDb: number; peakDb: number }
-  | { kind: 'pitch'; slot: number; positionMs: number; hz: number; confidence: number }
+  | { kind: 'pitch'; slot: number; positionMs: number; hz: number; confidence: number; targetMidi?: number; errorSemitones: number }
   | { kind: 'position'; frames: number }
   | { kind: 'backingEnded' }
   | { kind: 'underrun' }
@@ -139,6 +157,7 @@ export type NativeEvent =
   | { kind: 'retired' }
   | { kind: 'renderProgress'; job: number; fraction: number }
   | { kind: 'renderFinished'; job: number }
+  | { kind: 'outputChanged'; output: OutputInfo; gapMs: number }
 
 export interface NoteInput {
   midi: number
@@ -148,11 +167,19 @@ export interface NoteInput {
 
 export interface OpenOptions {
   clock: 'offline' | 'timer' | 'device'
-  /** A device clock's output: an id from `outputs`, or the default output. */
+  /** A device clock's output: an id from `outputs` to pin, or none to follow the default output. */
   device?: string
   periodFrames: number
-  /** The jitter buffer's playout delay for every `rtp` and `l16` slot. */
-  playoutDelayFrames: number
+  /**
+   * Pins the jitter buffer's playout delay for every `rtp` and `l16` slot. Without it, each
+   * stream chooses its own.
+   */
+  playoutDelayFrames?: number
+}
+
+export interface OutputDevice {
+  id: string
+  name: string
 }
 
 export interface OutputEntry {
@@ -165,6 +192,8 @@ export interface OutputInfo {
   sampleRate: number
   periodFrames: number
   latencyMs: number
+  /** The output playing now; none on the offline clock. */
+  device?: OutputDevice
 }
 
 export interface ReverbInput {
@@ -197,6 +226,10 @@ export interface SlotParamsInput {
   solo: boolean
   corrector: CorrectorInput
   reverb: ReverbInput
+  voice: VoiceInput
+  echoCancel: SwitchInput
+  cleanup: CleanupInput
+  howlGuard: SwitchInput
   guideOffsetMs: number
 }
 
@@ -205,14 +238,29 @@ export interface SlotStatsInfo {
   pushed: number
   lost: number
   late: number
+  lateAtOpening: number
   concealed: number
   resyncs: number
   jitterMs: number
   correctorDelayMs: number
   correctorMaxDelayMs: number
+  /** Null while the canceller is off or has yet to find and learn the room's echo. */
+  echo: EchoInfo | null
+  playoutDelayMs: number
+  howlCatches: number
 }
 
 export interface StemGainsInput {
   instrumental: number
   guide: number
 }
+
+/** An effect with no knob but on and off. */
+export interface SwitchInput {
+  enabled: boolean
+}
+
+export interface VoiceInput {
+  enabled: boolean
+  formantSemitones: number
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aeonentertainment/mixer",
-  "version": "0.3.0",
+  "version": "0.5.0",
   "description": "Low-latency LAN audio mixer for Aeon Karaoke: phone and house mics, a backing track, one mix to the room.",
   "bugs": {
     "url": "https://github.com/aeonentertainment/aeon-mixer/issues"
@@ -26,7 +26,7 @@
     "@napi-rs/cli": "^3.10.9",
     "@types/node": "^26.6.6",
     "typescript": "^7.0.2",
-    "vite-plus": "1.4.0"
+    "vite-plus": "1.5.1"
   },
   "devEngines": {
     "packageManager": {