@aeonentertainment/mixer 0.2.0 → 0.4.0

package/README.md

@@ -15,7 +15,7 @@ Each [release](https://github.com/aeonentertainment/aeon-mixer/releases) carries
 Depend on its URL, and the lockfile pins its hash:
 
 ```bash
-npm install https://github.com/aeonentertainment/aeon-mixer/releases/download/v0.2.0/aeonentertainment-mixer-0.2.0.tgz
+npm install https://github.com/aeonentertainment/aeon-mixer/releases/download/v0.4.0/aeonentertainment-mixer-0.4.0.tgz
 ```
 
 ## Use
@@ -51,7 +51,8 @@ await engine.close();
 - **Slots** are the mics: 8 per engine by default, up to 16 with `openEngine({ slots })`. `rtp`
   takes Opus over UDP from a phone or mediasoup, `l16` takes raw PCM over loopback from a house
   mic, and `push` takes PCM the host hands to `mixer.push`. Each slot has gain, pan, mute, solo,
-  reverb, a pitch reading and a melody guide.
+  reverb, a pitch reading, a melody guide and pitch correction toward it, whose delay `stats()`
+  reports.
 - **The backing** is any format Symphonia decodes, resampled to 48 kHz. The armed key and speed
   are rendered before `start` resolves. Live, it supports seek, pause, resume, a key change and
   stem gains, and `start`, `seek` and `resume` report the position the lyric clock follows.
@@ -62,6 +63,11 @@ await engine.close();
   card. It opens no device; its own thread renders a block at each deadline on the OS's
   monotonic clock, and the mix leaves on `rtpReturn`. `measure-clock` prints its timing:
   `cargo run --release -p devices --bin measure-clock` in `native/`.
+- **The output** is the OS default, unless `openEngine({ output: { device } })` pins one by an id
+  from `engine.outputDevices()`. When the default changes, or the output goes away, the song
+  moves to the new output after a short gap, a pinned engine moving back when its output
+  returns. `engine.output` names the output playing and its latency, and `outputChanged` fires
+  after each move with both and the gap.
 - **The offline engine**, `openEngine({ output: { offline: true } })`, opens no device and
   renders only when its caller runs `engine.pump(blocks)`. The tests run on it.
 

package/dist/index.d.mts

@@ -18,6 +18,7 @@ declare class Engine$1 {
    * gets its socket and send task here too.
    */
   createMixer(kinds: Array<string>, rtpReturn?: RtpReturnInput | undefined | null): Mixer$1;
+  /** The output playing now. After a move the events thread sends the same as `outputChanged`. */
   get output(): OutputInfo$1;
   /** The outputs this OS offers, the default marked. Empty on the offline and timer clocks. */
   outputs(): Array<OutputEntry>;
@@ -72,7 +73,7 @@ declare class Mixer$1 {
    * recording it for null.
    */
   recordSlot(index: number, path?: string | undefined | null): void;
-  /** Every slot's ingest counters, as the audio thread last published them. */
+  /** Every slot's ingest counters and corrector delay, as the audio thread last published them. */
   stats(): Array<SlotStatsInfo>;
   /** The return stream's counters, or null for a mixer without one. */
   returnStats(): RtpReturnStats | null;
@@ -119,6 +120,8 @@ type NativeEvent = {
   positionMs: number;
   hz: number;
   confidence: number;
+  targetMidi?: number;
+  errorSemitones: number;
 } | {
   kind: 'position';
   frames: number;
@@ -137,6 +140,10 @@ type NativeEvent = {
 } | {
   kind: 'renderFinished';
   job: number;
+} | {
+  kind: 'outputChanged';
+  output: OutputInfo$1;
+  gapMs: number;
 };
 interface NoteInput {
   midi: number;
@@ -145,12 +152,16 @@ interface NoteInput {
 }
 interface OpenOptions {
   clock: 'offline' | 'timer' | 'device';
-  /** A device clock's output: an id from `outputs`, or the default output. */
+  /** A device clock's output: an id from `outputs` to pin, or none to follow the default output. */
   device?: string;
   periodFrames: number;
   /** The jitter buffer's playout delay for every `rtp` and `l16` slot. */
   playoutDelayFrames: number;
 }
+interface OutputDevice {
+  id: string;
+  name: string;
+}
 interface OutputEntry {
   id: string;
   name: string;
@@ -160,6 +171,8 @@ interface OutputInfo$1 {
   sampleRate: number;
   periodFrames: number;
   latencyMs: number;
+  /** The output playing now; none on the offline clock. */
+  device?: OutputDevice;
 }
 interface ReverbInput {
   enabled: boolean;
@@ -188,6 +201,7 @@ interface SlotParamsInput {
   solo: boolean;
   corrector: CorrectorInput;
   reverb: ReverbInput;
+  voice: VoiceInput;
   guideOffsetMs: number;
 }
 /** `SlotStats` as JavaScript reads it. */
@@ -198,11 +212,17 @@ interface SlotStatsInfo {
   concealed: number;
   resyncs: number;
   jitterMs: number;
+  correctorDelayMs: number;
+  correctorMaxDelayMs: number;
 }
 interface StemGainsInput {
   instrumental: number;
   guide: number;
 }
+interface VoiceInput {
+  enabled: boolean;
+  formantSemitones: number;
+}
 declare const NativeEngine: typeof Engine$1;
 type NativeEngine = Engine$1;
 type NativeMixer = Mixer$1;
@@ -211,8 +231,9 @@ type IngestKind = "rtp" | "l16" | "push";
 type StemKind = "instrumental" | "guideVocals";
 interface OpenEngineOptions {
   /**
-   * A device; `timer` for a box with no sound card, whose own thread renders on the OS's clock
-   * and whose mix leaves only on a mixer's `rtpReturn`; `offline` for tests, which pump it.
+   * A device, pinned by `device` or, left out, following the OS default; `timer` for a box with
+   * no sound card, whose own thread renders on the OS's clock and whose mix leaves only on a
+   * mixer's `rtpReturn`; `offline` for tests, which pump it.
    */
   output: {
     device?: string;
@@ -231,6 +252,11 @@ interface OutputInfo {
   readonly sampleRate: 48000;
   readonly periodFrames: number;
   readonly latencyMs: number;
+  /** The output playing now, as `outputDevices()` lists it; null on the offline clock. */
+  readonly device: {
+    readonly id: string;
+    readonly name: string;
+  } | null;
 }
 interface MixerConfig {
   slots: ReadonlyArray<{
@@ -273,6 +299,15 @@ type SlotEffect = {
   enabled: boolean;
   roomSize: number;
   mix: number;
+} |
+/**
+ * A formant move in semitones from the singer's own, -12 to 12, with no change of pitch and no
+ * added delay. Decision 0014 has the first voices' settings.
+ */
+{
+  kind: "voice";
+  enabled: boolean;
+  formantSemitones: number;
 };
 interface MelodyGuide {
   notes: ReadonlyArray<{
@@ -290,15 +325,31 @@ interface SlotStats {
   concealed: number;
   resyncs: number;
   jitterMs: number;
+  /**
+   * How far the corrector holds this slot's voice now; 0 with correction off. It moves with the
+   * voice and one read is a sample of it, so show it and never add it to a budget or a clock.
+   */
+  correctorDelayMs: number;
+  /** The most that delay can reach; 0 with correction off. It holds still, so a budget adds this. */
+  correctorMaxDelayMs: number;
 }
 type EngineEvents = {
   fault: (error: Error) => void;
+  /** After every move to another output; `gapMs` is how long the room heard nothing. */
+  outputChanged: (output: OutputInfo, gapMs: number) => void;
   /** The mix after the clipper, reactions included, between songs too. */
   masterLevel: (rmsDb: number, peakDb: number) => void;
 };
 type MixerEvents = {
   level: (slot: number, rmsDb: number, peakDb: number) => void;
-  pitch: (slot: number, positionMs: number, hz: number, confidence: number) => void;
+  /**
+   * Every 10 ms, on the guide's clock. `targetMidi` is the guide note the corrector pulls toward,
+   * after the backing's key shift: null with correction off, with no guide, or between notes,
+   * where it pulls toward the key. `errorSemitones` is the class it pulls toward less the
+   * reading, in the singer's own octave, so positive is flat: 0 with correction off or while
+   * unvoiced.
+   */
+  pitch: (slot: number, positionMs: number, hz: number, confidence: number, targetMidi: number | null, errorSemitones: number) => void;
   position: (positionMs: number) => void;
   renderProgress: (fraction: number) => void;
   backingEnded: () => void;
@@ -323,7 +374,8 @@ declare class Emitter<Events extends Record<keyof Events, Handler>> {
 }
 /** What a mixer needs from its engine. Internal to the package. */
 interface MixerHost extends RenderHost {
-  readonly output: OutputInfo;
+  /** The output playing now; it changes when the engine moves. */
+  output(): OutputInfo;
   /**
    * Sends a command carrying an id and resolves on its ack with the backing position. `onTimeout`
    * hears the rejection when no ack comes in time.
@@ -441,10 +493,14 @@ export declare function openEngine(options: OpenEngineOptions): Promise<Engine>;
 export declare class Engine {
   #private;
   readonly capabilities: ReadonlySet<Capability>;
-  readonly output: OutputInfo;
   /** Offline engines only: render blocks and return them as interleaved stereo. */
   readonly pump?: (blocks: number) => Float32Array;
   constructor(options: OpenEngineOptions);
+  /**
+   * The output playing now. An engine opened with no `output.device` follows the OS default, and
+   * one with a `device` plays there while it exists; `outputChanged` fires after every move.
+   */
+  get output(): OutputInfo;
   /** Room-lifetime reaction voices, four at once. False when a fifth is dropped while four play. */
   readonly reactions: {
     play: (samples: Float32Array, channels: 1 | 2) => boolean;

package/dist/index.mjs

@@ -230,6 +230,10 @@ function defaultSlotParams() {
 			roomSize: 0,
 			mix: 0
 		},
+		voice: {
+			enabled: false,
+			formantSemitones: 0
+		},
 		guideOffsetMs: 0
 	};
 }
@@ -305,10 +309,16 @@ var Mixer = class {
 					retuneMs: finiteIn("retuneMs", effect.retuneMs, 0, 1e3)
 				};
 				break;
-			case "reverb": next.reverb = {
+			case "reverb":
+				next.reverb = {
+					enabled: effect.enabled,
+					roomSize: finiteIn("roomSize", effect.roomSize, 0, 1),
+					mix: finiteIn("mix", effect.mix, 0, 1)
+				};
+				break;
+			case "voice": next.voice = {
 				enabled: effect.enabled,
-				roomSize: finiteIn("roomSize", effect.roomSize, 0, 1),
-				mix: finiteIn("mix", effect.mix, 0, 1)
+				formantSemitones: finiteIn("formantSemitones", effect.formantSemitones, -12, 12)
 			};
 		}
 		this.#write(slot, next);
@@ -393,7 +403,7 @@ var Mixer = class {
 			started,
 			stopping
 		]);
-		const { latencyMs } = this.#host.output;
+		const { latencyMs } = this.#host.output();
 		return {
 			startedAt: Date.now() + latencyMs,
 			backingOffsetMs: latencyMs,
@@ -488,10 +498,11 @@ var Mixer = class {
 	}
 };
 function defaultEffects() {
-	const { corrector, reverb } = defaultSlotParams();
+	const { corrector, reverb, voice } = defaultSlotParams();
 	return {
 		corrector,
-		reverb
+		reverb,
+		voice
 	};
 }
 function toGuideInput(guide) {
@@ -522,7 +533,7 @@ var Engine = class {
 		"pause",
 		"level"
 	]);
-	output;
+	#output;
 	/** How many slots a mixer may have, fixed at open. */
 	#slots;
 	/** Offline engines only: render blocks and return them as interleaved stereo. */
@@ -566,10 +577,10 @@ var Engine = class {
 				...device === void 0 ? {} : { device }
 			}, deliver);
 		}
-		this.output = outputInfo(this.#native.output);
+		this.#output = outputInfo(this.#native.output);
 		this.#host = {
 			native: this.#native,
-			output: this.output,
+			output: () => this.#output,
 			command: (send, onTimeout) => this.#command(send, onTimeout),
 			watchRender: (id, onProgress) => new Promise((resolve, reject) => {
 				this.#renders.set(id, {
@@ -586,6 +597,13 @@ var Engine = class {
 			underruns: () => this.#underruns
 		};
 	}
+	/**
+	* The output playing now. An engine opened with no `output.device` follows the OS default, and
+	* one with a `device` plays there while it exists; `outputChanged` fires after every move.
+	*/
+	get output() {
+		return this.#output;
+	}
 	/** Room-lifetime reaction voices, four at once. False when a fifth is dropped while four play. */
 	reactions = { play: (samples, channels) => {
 		if (channels !== 1 && channels !== 2) throw new RangeError(`channels must be 1 or 2, got ${String(channels)}`);
@@ -654,7 +672,7 @@ var Engine = class {
 				this.#live?.events.emit("level", event.slot, event.rmsDb, event.peakDb);
 				break;
 			case "pitch":
-				this.#live?.events.emit("pitch", event.slot, event.positionMs, event.hz, event.confidence);
+				this.#live?.events.emit("pitch", event.slot, event.positionMs, event.hz, event.confidence, event.targetMidi ?? null, event.errorSemitones);
 				break;
 			case "position":
 				this.#live?.events.emit("position", framesToMs(event.frames));
@@ -677,6 +695,10 @@ var Engine = class {
 			case "renderProgress":
 				this.#renders.get(event.job)?.onProgress(event.fraction);
 				break;
+			case "outputChanged":
+				this.#output = outputInfo(event.output);
+				this.#events.emit("outputChanged", this.#output, event.gapMs);
+				break;
 			case "renderFinished": {
 				const render = this.#renders.get(event.job);
 				this.#renders.delete(event.job);
@@ -705,10 +727,15 @@ var Engine = class {
 };
 function outputInfo(info) {
 	if (info.sampleRate !== 48e3) throw new Error(`the engine runs at 48 kHz; the addon reports ${info.sampleRate}`);
+	const { periodFrames, latencyMs, device } = info;
 	return {
 		sampleRate: 48e3,
-		periodFrames: info.periodFrames,
-		latencyMs: info.latencyMs
+		periodFrames,
+		latencyMs,
+		device: device === void 0 ? null : {
+			id: device.id,
+			name: device.name
+		}
 	};
 }
 function monoToStereo(mono) {

package/native/binding/index.d.ts

@@ -30,6 +30,7 @@ export declare class Engine {
    * gets its socket and send task here too.
    */
   createMixer(kinds: Array<string>, rtpReturn?: RtpReturnInput | undefined | null): Mixer
+  /** The output playing now. After a move the events thread sends the same as `outputChanged`. */
   get output(): OutputInfo
   /** The outputs this OS offers, the default marked. Empty on the offline and timer clocks. */
   outputs(): Array<OutputEntry>
@@ -85,7 +86,7 @@ export declare class Mixer {
    * recording it for null.
    */
   recordSlot(index: number, path?: string | undefined | null): void
-  /** Every slot's ingest counters, as the audio thread last published them. */
+  /** Every slot's ingest counters and corrector delay, as the audio thread last published them. */
   stats(): Array<SlotStatsInfo>
   /** The return stream's counters, or null for a mixer without one. */
   returnStats(): RtpReturnStats | null
@@ -131,7 +132,7 @@ export type NativeEvent =
   | { kind: 'ack'; id: number; positionFrames: number }
   | { kind: 'level'; slot: number; rmsDb: number; peakDb: number }
   | { kind: 'masterLevel'; rmsDb: number; peakDb: number }
-  | { kind: 'pitch'; slot: number; positionMs: number; hz: number; confidence: number }
+  | { kind: 'pitch'; slot: number; positionMs: number; hz: number; confidence: number; targetMidi?: number; errorSemitones: number }
   | { kind: 'position'; frames: number }
   | { kind: 'backingEnded' }
   | { kind: 'underrun' }
@@ -139,6 +140,7 @@ export type NativeEvent =
   | { kind: 'retired' }
   | { kind: 'renderProgress'; job: number; fraction: number }
   | { kind: 'renderFinished'; job: number }
+  | { kind: 'outputChanged'; output: OutputInfo; gapMs: number }
 
 export interface NoteInput {
   midi: number
@@ -148,13 +150,18 @@ export interface NoteInput {
 
 export interface OpenOptions {
   clock: 'offline' | 'timer' | 'device'
-  /** A device clock's output: an id from `outputs`, or the default output. */
+  /** A device clock's output: an id from `outputs` to pin, or none to follow the default output. */
   device?: string
   periodFrames: number
   /** The jitter buffer's playout delay for every `rtp` and `l16` slot. */
   playoutDelayFrames: number
 }
 
+export interface OutputDevice {
+  id: string
+  name: string
+}
+
 export interface OutputEntry {
   id: string
   name: string
@@ -165,6 +172,8 @@ export interface OutputInfo {
   sampleRate: number
   periodFrames: number
   latencyMs: number
+  /** The output playing now; none on the offline clock. */
+  device?: OutputDevice
 }
 
 export interface ReverbInput {
@@ -197,6 +206,7 @@ export interface SlotParamsInput {
   solo: boolean
   corrector: CorrectorInput
   reverb: ReverbInput
+  voice: VoiceInput
   guideOffsetMs: number
 }
 
@@ -208,9 +218,16 @@ export interface SlotStatsInfo {
   concealed: number
   resyncs: number
   jitterMs: number
+  correctorDelayMs: number
+  correctorMaxDelayMs: number
 }
 
 export interface StemGainsInput {
   instrumental: number
   guide: number
 }
+
+export interface VoiceInput {
+  enabled: boolean
+  formantSemitones: number
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aeonentertainment/mixer",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Low-latency LAN audio mixer for Aeon Karaoke: phone and house mics, a backing track, one mix to the room.",
   "bugs": {
     "url": "https://github.com/aeonentertainment/aeon-mixer/issues"