@lumavox/client 0.15.1 → 0.17.0

package/README.md

@@ -143,12 +143,37 @@ startButton.onclick = () => audio.prime(); // in the click: browsers start sound
 ```
 
 An `AgentAudio` on Web Audio, its own import so the client carries none of
-it (~5.5 KB minified): replies play gaplessly as they arrive, at the
+it (~8.5 KB minified): replies play gaplessly as they arrive, at the
 device's own rate; the microphone records at 24 kHz, with the browser's
 echo cancellation, noise suppression and gain control. Hand it to
 `@lumavox/react`'s `useAgent`, or call `play`, `startCapture` and the rest
 yourself.
 
+**The clock**, to draw a face or subtitles against the voice:
+
+```ts
+const audio = createWebAudio({ leadInMs: 350 }); // time to read each frame before it plays
+audio.onScheduled(({ pcm, sample_rate, startsAt }) => lipsync.add(pcm, sample_rate, startsAt));
+audio.onStopped(() => lipsync.clear()); // a cancel, a mute, a barge-in
+// each animation frame: lipsync.at(audio.heardTime())
+```
+
+- `onScheduled`: every frame as it is queued, its 16-bit samples and when
+  they start on `now()`'s clock. `heardTime()`: what is heard at this
+  moment, the clock less the browser's output latency (Bluetooth adds to
+  it).
+- `leadInMs`: a reply's first frame, or one after a gap, waits that long —
+  60 ms unless set; it adds as much to the first word.
+- `onStopped`: a `stop()` that dropped audio still to play. `onCapture((pcm,
+  rate) => …)`: every frame the microphone captures, for a level meter.
+
+**The devices**: `inputs()` and `setInput(id)` for the microphone (one
+unplugged falls back to the default; labels are empty until it was
+allowed), `setProcessing({ echoCancellation, noiseSuppression,
+autoGainControl })` — each on unless `false`, and hands-free refused with
+echo cancellation off — and, where `canChooseOutput` (Chrome and Edge),
+`outputs()` and `setOutputDevice(id)` for the speaker.
+
 ### Hands-free
 
 ```ts
@@ -166,6 +191,13 @@ allows it where the browser cancels echo (not on an iPhone or iPad);
 `assumeNoEcho` with headphones on. `loudness()` is the built-in detector,
 `detector` your own.
 
+**Half-duplex**, where echo cannot be cancelled: `listen(conversation,
+audio, { halfDuplex: true })` keeps the microphone open but sends nothing
+while the agent's voice is heard — on the player's clock, then 300 ms more
+(`afterVoiceMs`) for the room's echo — and talking never stops it, so the
+agent never hears itself and it is never refused for echo. Something said
+as the voice starts ends there as a turn, answered after the reply.
+
 ## Runtimes
 
 Browsers, React Native (Hermes, ES2020 — and each binary frame reaches JS

package/dist/audio.d.ts

@@ -8,7 +8,11 @@
  * mute or a new turn, records at the rate the recorder was granted, and
  * throws away a recording the device interrupted. Hands-free — the
  * microphone left open, a detector saying when the person talks — is
- * `listen` (`./hands-free.ts`), on the two optional methods below.
+ * `listen` (`./hands-free.ts`), on the two optional methods below. The
+ * clock, the frames as they are queued and captured, and the devices
+ * (`PlaybackClock`, `DeviceChoices`) are the two adapters' own, for the
+ * app: nothing that drives an `AgentAudio` needs them, so an adapter of an
+ * app's own need not have them.
  */
 export interface AgentAudio {
     /** Queue one frame of a reply: 16-bit little-endian PCM, in wire order. */
@@ -30,6 +34,12 @@ export interface AgentAudio {
      */
     startCapture(onPcm: (pcm: ArrayBuffer) => void, options?: {
         rate?: number;
+        /**
+         * Hands-free asks it, once `handsFree` said yes: take the agent's
+         * voice out of the microphone — a phone's voice processing and call
+         * audio. An adapter that always does, as a browser's, ignores it.
+         */
+        echoCancellation?: boolean;
     }): Promise<{
         rate: number;
     }>;
@@ -62,3 +72,86 @@ export interface AgentAudio {
     /** Optional: play the reply quieter, or as before — barge-in's first step, on sound alone. */
     duck?(on: boolean): void;
 }
+/** One frame of a reply as the player queued it: what a face or subtitles are drawn against. */
+export interface ScheduledAudio {
+    /** The 16-bit little-endian samples queued: a sample split across two frames already joined. */
+    pcm: ArrayBuffer;
+    sample_rate: number;
+    channels: number;
+    /** As the socket sent it: the text chunk this is the voice of, in its turn. */
+    chunk: number;
+    turn: string;
+    /** When it starts to play, in seconds on `now()`'s clock. */
+    startsAt: number;
+    /** Seconds. */
+    duration: number;
+}
+/**
+ * The player's clock and what passes through it, for an app that draws
+ * against the voice — a face lip-synced, subtitles revealed chunk by chunk
+ * — or against the microphone, a level meter. `@lumavox/client/web-audio`'s
+ * and `@lumavox/react-native`'s audio have it; `useAgent` needs none of it.
+ */
+export interface PlaybackClock {
+    /** The player's clock, in seconds: its audio context's. `0` before `prime()`. */
+    now(): number;
+    /**
+     * What the listener hears at this moment, on `now()`'s clock: `now()`
+     * less the time the sound takes to leave the device, which Bluetooth
+     * makes longer. Never negative.
+     */
+    heardTime(): number;
+    /**
+     * Every frame as it is queued, before it plays: its samples and when
+     * they start. A handler that throws is rethrown on its own. Answers the
+     * unsubscribe.
+     */
+    onScheduled(fn: (audio: ScheduledAudio) => void): () => void;
+    /**
+     * When `stop()` drops audio still to play — a cancel, a mute, a
+     * barge-in, a new turn. `onDrained` fires then too, and when a reply
+     * plays out. Answers the unsubscribe.
+     */
+    onStopped(fn: () => void): () => void;
+    /**
+     * Every frame the microphone captures while it is open — what
+     * `startCapture` hands on, and what hands-free does not send — as 16-bit
+     * PCM at `rate`. Answers the unsubscribe.
+     */
+    onCapture(fn: (pcm: ArrayBuffer, rate: number) => void): () => void;
+}
+/** A microphone or a speaker, as the browser lists it. */
+export interface AudioDevice {
+    id: string;
+    /** Empty until the person has allowed the microphone. */
+    label: string;
+}
+/** The browser's own cleanup of the microphone: each on unless `false`. */
+export interface Processing {
+    /** Off, hands-free is refused: the agent would hear itself. */
+    echoCancellation?: boolean;
+    noiseSuppression?: boolean;
+    autoGainControl?: boolean;
+}
+/**
+ * The microphone and the speaker a browser lets a page choose. On a phone
+ * the system chooses: no devices are listed and none can be set.
+ */
+export interface DeviceChoices {
+    /** The microphones; labels are empty until the microphone was allowed. */
+    inputs(): Promise<AudioDevice[]>;
+    /** The microphone, by its id, from the next capture; none, the default. One unplugged since falls back to the default. */
+    setInput(id?: string): void;
+    /** The browser's processing from the next capture: what is left out is on. */
+    setProcessing(processing: Processing): void;
+    /** The speakers a page can play through; `[]` where it cannot choose. */
+    outputs(): Promise<AudioDevice[]>;
+    /**
+     * Play through the speaker `id`, kept for a player made later. Answers
+     * whether it took: `false` where the browser cannot choose, or the
+     * speaker is gone.
+     */
+    setOutputDevice(id: string): Promise<boolean>;
+    /** Whether this browser lets a page choose its speaker: Chrome and Edge. */
+    readonly canChooseOutput: boolean;
+}

package/dist/hands-free.d.ts

@@ -25,10 +25,36 @@
  *   cancellation leaves of its voice is louder than a quiet room.
  * - **Refused where the agent would hear itself** (D16): an adapter that
  *   says so, or cannot say (`AgentAudio.handsFree`) unless the app knows.
+ *   On its yes the recorder is asked to cancel the echo
+ *   (`startCapture`'s `echoCancellation`): a phone's is a mode of its own —
+ *   the call's audio, quieter — never wanted with headphones on or in
+ *   half-duplex.
+ * - **Half-duplex** (D19, `halfDuplex`), where there is no echo to cancel —
+ *   a phone on `react-native-audio-api` 0.13, a browser that cannot cancel
+ *   it: the microphone stays open, but **while the agent's voice is heard
+ *   nothing is sent, the detector rests and there is no barge-in**, so the
+ *   agent never hears itself and `handsFree()` is not asked. Heard is the
+ *   player's clock (`PlaybackClock`: each frame's `startsAt` and
+ *   `duration`, against `heardTime()`), never when frames are queued: a
+ *   phone queues a reply seconds ahead of its sound. An adapter with no
+ *   clock is taken at its word, from an `audio.start` to `onDrained`.
+ *   **The voice has the floor**: an utterance open when it starts ends
+ *   there as a turn — what was said before it is kept, and answered after
+ *   the reply. **Then `afterVoiceMs`, 300 ms, before the detector listens
+ *   again**: what the clock cannot know of a phone's sound path (0.13 says
+ *   none; most phones' loudspeakers take 20 to 100 ms), the microphone's own
+ *   latency (10 to 50 ms), and the room's echo falling 30 dB (half of a
+ *   furnished room's 0.3 to 0.6 s reverberation) — else the agent's last
+ *   syllable, ringing on, opens a recording of itself. It costs no reply
+ *   any time: those frames are kept as pre-roll, so words begun in it keep
+ *   their start; only a word over before it ends — a quick "yes" under
+ *   0.3 s — goes unheard.
  *
  * Time here is audio time, counted in 20 ms frames, never the clock's: a
  * busy thread delivers late, and the frames still say how long the
- * silence was.
+ * silence was. Half-duplex alone reads the player's clock, to set each
+ * frame against the voice: the last sample delivered is heard now, and
+ * each before it a sample earlier.
  */
 import type { AgentAudio } from "./audio.js";
 import type { Conversation, TurnOptions } from "./conversation.js";
@@ -88,6 +114,19 @@ export interface ListenOptions extends TurnOptions {
     wordsWithinMs?: number;
     /** The app knows the agent's voice stays out of the microphone — headphones, say — though the adapter cannot tell. */
     assumeNoEcho?: boolean;
+    /**
+     * Half-duplex: the microphone stays open, but nothing is sent while the
+     * agent's voice is heard, and talking never stops it — for where echo
+     * cannot be cancelled, such as a phone, so `handsFree()` is not asked.
+     */
+    halfDuplex?: boolean;
+    /**
+     * Half-duplex: after the agent's voice is heard ending, how long the
+     * detector still waits, ms: 300 — so the room's echo of its last word
+     * does not open a recording. Words begun in it keep their start; one
+     * over before it ends is not heard. Lower with headphones on.
+     */
+    afterVoiceMs?: number;
     /** `listening` for the person, `hearing` them, `paused` while the device has the audio, then `stopped`. */
     onState?(state: ListeningState): void;
     /** The person talked over the agent: its voice was stopped. */
@@ -107,12 +146,13 @@ export type ListenOutcome = {
     reason: string;
 };
 /** Why `listen` refuses an adapter that cannot say whether the agent hears itself. */
-export declare const NO_ECHO_ANSWER = "This audio adapter cannot say whether the agent's voice stays out of the microphone (`AgentAudio.handsFree`). Use push-to-talk, or pass `assumeNoEcho` if you know it does.";
+export declare const NO_ECHO_ANSWER = "This audio adapter cannot say whether the agent's voice stays out of the microphone (`AgentAudio.handsFree`). Use half-duplex (`halfDuplex`) or push-to-talk, or pass `assumeNoEcho` if you know it does.";
 /**
  * Leave the microphone open on `conversation`: every utterance the
- * detector hears becomes a turn, and talking over the agent stops it.
- * Answers `ok: false` with the reason, in words an app can show, where the
- * agent would hear itself or the microphone would not open — push-to-talk
- * still works there.
+ * detector hears becomes a turn, and talking over the agent stops it — or,
+ * with `halfDuplex`, nothing is heard while the agent speaks. Answers
+ * `ok: false` with the reason, in words an app can show, where the agent
+ * would hear itself or the microphone would not open — half-duplex and
+ * push-to-talk still work there.
  */
 export declare function listen(conversation: Conversation, audio: AgentAudio, options?: ListenOptions): Promise<ListenOutcome>;

package/dist/hands-free.js

@@ -40,16 +40,17 @@ export function loudness(options = {}) {
     };
 }
 /** Why `listen` refuses an adapter that cannot say whether the agent hears itself. */
-export const NO_ECHO_ANSWER = "This audio adapter cannot say whether the agent's voice stays out of the microphone (`AgentAudio.handsFree`). Use push-to-talk, or pass `assumeNoEcho` if you know it does.";
+export const NO_ECHO_ANSWER = "This audio adapter cannot say whether the agent's voice stays out of the microphone (`AgentAudio.handsFree`). Use half-duplex (`halfDuplex`) or push-to-talk, or pass `assumeNoEcho` if you know it does.";
 /**
  * Leave the microphone open on `conversation`: every utterance the
- * detector hears becomes a turn, and talking over the agent stops it.
- * Answers `ok: false` with the reason, in words an app can show, where the
- * agent would hear itself or the microphone would not open — push-to-talk
- * still works there.
+ * detector hears becomes a turn, and talking over the agent stops it — or,
+ * with `halfDuplex`, nothing is heard while the agent speaks. Answers
+ * `ok: false` with the reason, in words an app can show, where the agent
+ * would hear itself or the microphone would not open — half-duplex and
+ * push-to-talk still work there.
  */
 export async function listen(conversation, audio, options = {}) {
-    if (!options.assumeNoEcho) {
+    if (!options.assumeNoEcho && !options.halfDuplex) {
         const echo = audio.handsFree
             ? await audio.handsFree()
             : { ok: false, reason: NO_ECHO_ANSWER };
@@ -80,6 +81,12 @@ class HandsFree {
         this.playingTurn = null;
         /** The reply a barge-in stopped: its late windows are not the agent speaking. */
         this.stoppedTurn = null;
+        /** Half-duplex, on that clock: the agent's voice as scheduled, in runs of frames back to back. */
+        this.voice = [];
+        /** The clock as last read, to see it go back. */
+        this.read = 0;
+        /** Half-duplex with no clock: audio time since the voice was last playing. */
+        this.sinceVoiceMs = Number.POSITIVE_INFINITY;
         this.off = [];
         this.conversation = conversation;
         this.audio = audio;
@@ -89,12 +96,18 @@ class HandsFree {
         this.startAfterMs = options.startAfterMs ?? 100;
         this.wordsWithinMs = options.wordsWithinMs ?? 3000;
         this.ringFrames = Math.ceil(((options.preRollMs ?? 300) + this.startAfterMs) / FRAME_MS);
+        this.halfDuplex = options.halfDuplex === true;
+        this.afterVoiceMs = options.afterVoiceMs ?? 300;
+        this.clock = this.halfDuplex ? clockOf(audio) : null;
     }
     /** The microphone, and everything watched: `null`, or why it did not open. */
     async start() {
         this.off.push(this.conversation.onFrame((frame) => this.frame(frame)), this.audio.onDrained(() => {
             this.playing = false;
         }));
+        if (this.clock) {
+            this.off.push(this.clock.onScheduled((audio) => this.scheduled(audio)), this.clock.onStopped(() => this.cut()));
+        }
         const interrupted = this.audio.onInterruption?.((now) => this.interrupted(now));
         if (interrupted)
             this.off.push(interrupted);
@@ -119,7 +132,14 @@ class HandsFree {
     async capture() {
         let granted;
         try {
-            ({ rate: granted } = await this.audio.startCapture((pcm) => this.pcm(pcm), { rate: this.options.rate ?? 16000 }));
+            ({ rate: granted } = await this.audio.startCapture((pcm) => this.pcm(pcm), {
+                rate: this.options.rate ?? 16000,
+                // On the adapter's word that it can: half-duplex and the app's
+                // own word need none, and a phone's sounds better without.
+                ...(this.options.halfDuplex || this.options.assumeNoEcho
+                    ? {}
+                    : { echoCancellation: true }),
+            }));
         }
         catch (failure) {
             // swallowed: it is the answer's reason, the app's to show.
@@ -170,11 +190,16 @@ class HandsFree {
         void this.capture().then((failed) => {
             if (this.state !== "paused")
                 return;
-            // Still taken, or gone: stopped, and the app's next `listen` tries again.
-            if (failed === null)
+            if (failed === null) {
                 this.set("listening");
-            else
-                this.close();
+                return;
+            }
+            // Still taken, or gone: stopped, and the app's next `listen` tries
+            // again. Given back, so what the capture set up is undone.
+            this.close();
+            this.audio.stopCapture().catch(() => {
+                // swallowed: nothing was open; the adapter has let go either way.
+            });
         });
     }
     pcm(buffer) {
@@ -187,14 +212,25 @@ class HandsFree {
             samples.set(this.carry, 0);
             samples.set(fresh, this.carry.length);
         }
+        // Half-duplex: the last sample is heard now, on the player's clock.
+        const heard = this.clock ? this.heardNow() : 0;
         let at = 0;
         for (; at + this.frameSamples <= samples.length; at += this.frameSamples) {
-            this.step(samples.slice(at, at + this.frameSamples));
+            const end = at + this.frameSamples;
+            this.step(samples.slice(at, end), heard - (samples.length - end) / this.rate);
         }
         this.carry = samples.slice(at);
     }
-    step(frame) {
-        const voice = this.detector.push(frame, this.rate, this.playing);
+    /** One 20 ms frame; `endsAt`, half-duplex on a clock, when its last sample was heard. */
+    step(frame, endsAt) {
+        if (this.halfDuplex) {
+            const ago = this.voiceAgo(endsAt);
+            if (ago === 0 || ago < this.afterVoiceMs) {
+                this.voiceHeard(ago === 0 ? null : frame);
+                return;
+            }
+        }
+        const voice = this.detector.push(frame, this.rate, this.playing && !this.halfDuplex);
         const talking = voice === "talking";
         if (talking) {
             this.talkMs += FRAME_MS;
@@ -206,9 +242,7 @@ class HandsFree {
         }
         const open = this.open;
         if (!open) {
-            this.ring.push(frame);
-            if (this.ring.length > this.ringFrames)
-                this.ring.shift();
+            this.keep(frame);
             if (this.waitForQuiet && this.quietMs >= this.endAfterMs) {
                 this.waitForQuiet = false;
             }
@@ -228,12 +262,96 @@ class HandsFree {
             this.end();
         }
     }
+    /** A frame of the room kept for the next utterance's pre-roll. */
+    keep(frame) {
+        this.ring.push(frame);
+        if (this.ring.length > this.ringFrames)
+            this.ring.shift();
+    }
+    /**
+     * Half-duplex: ms between the agent's voice last heard and the frame
+     * ending at `endsAt` — 0 while it is heard, infinite before it ever was.
+     */
+    voiceAgo(endsAt) {
+        if (!this.clock) {
+            // Taken at the adapter's word: playing from `audio.start` to `onDrained`.
+            this.sinceVoiceMs = this.playing ? 0 : this.sinceVoiceMs + FRAME_MS;
+            return this.playing ? 0 : this.sinceVoiceMs;
+        }
+        const startsAt = endsAt - FRAME_MS / 1000;
+        let ago = Number.POSITIVE_INFINITY;
+        for (const run of this.voice) {
+            if (run.from < endsAt && run.until > startsAt)
+                return 0;
+            if (run.until <= startsAt)
+                ago = (startsAt - run.until) * 1000;
+        }
+        return ago;
+    }
+    /**
+     * Half-duplex: the agent's voice is heard, or its tail (`tail`, the
+     * frame). The voice has the floor — an utterance open ends as a turn
+     * before any of it is sent — and the detector rests; nothing of the
+     * voice is pre-roll, its tail is.
+     */
+    voiceHeard(tail) {
+        if (this.open)
+            this.end();
+        this.talkMs = 0;
+        this.quietMs = 0;
+        if (tail)
+            this.keep(tail);
+        else
+            this.ring = [];
+    }
+    /** The clock now; one that went back — a new context, the phone's for a reply at another rate — leaves the runs kept on the old one. */
+    now() {
+        const now = this.clock.now();
+        // The stop that came with it was told; the new run starts at once.
+        if (now < this.read)
+            this.voice = [];
+        this.read = now;
+        return now;
+    }
+    /** What is heard now on the clock, the runs long past forgotten. */
+    heardNow() {
+        this.now();
+        const heard = this.clock.heardTime();
+        const past = heard - (this.afterVoiceMs + 1000) / 1000;
+        while (this.voice.length > 1 && (this.voice[0]?.until ?? 0) < past) {
+            this.voice.shift();
+        }
+        return heard;
+    }
+    /** A frame of the reply, queued to start at `startsAt`: its run of the voice, or a new one after a gap. */
+    scheduled(audio) {
+        this.now();
+        const until = audio.startsAt + audio.duration;
+        const last = this.voice[this.voice.length - 1];
+        // Back to back with the one before, to a millisecond's rounding.
+        if (last &&
+            audio.startsAt >= last.from &&
+            audio.startsAt <= last.until + 0.001) {
+            last.until = Math.max(last.until, until);
+        }
+        else {
+            this.voice.push({ from: audio.startsAt, until });
+        }
+    }
+    /** The player dropped what it had queued: the voice ends where the clock is. */
+    cut() {
+        const now = this.now();
+        this.voice = this.voice.filter((run) => run.from < now);
+        for (const run of this.voice)
+            run.until = Math.min(run.until, now);
+    }
     begin() {
-        const barging = this.playing;
+        // Half-duplex never opens over the voice: no barge-in to decide.
+        const barging = this.playing && !this.halfDuplex;
         const options = this.options;
         const id = this.conversation.startAudio({
             rate: this.rate,
-            bargeIn: true,
+            bargeIn: !this.halfDuplex,
             ...(options.language ? { language: options.language } : {}),
             ...(options.model ? { model: options.model } : {}),
             ...(options.speech === undefined ? {} : { speech: options.speech }),
@@ -273,6 +391,16 @@ class HandsFree {
         this.options.onState?.(state);
     }
 }
+/** The adapter's `PlaybackClock`, when it has one: both of this SDK's do; one of an app's own need not. */
+function clockOf(audio) {
+    const clock = audio;
+    return typeof clock.now === "function" &&
+        typeof clock.heardTime === "function" &&
+        typeof clock.onScheduled === "function" &&
+        typeof clock.onStopped === "function"
+        ? audio
+        : null;
+}
 /** 16-bit little-endian PCM as samples, on a big-endian device too. */
 function samplesOf(buffer) {
     const count = buffer.byteLength >> 1;

package/dist/index.d.ts

@@ -7,7 +7,7 @@
  * vocabulary (`src/protocol.gen.ts`); methods are camelCase, and the SDK's
  * own options go last.
  */
-export type { AgentAudio } from "./audio.js";
+export type { AgentAudio, AudioDevice, DeviceChoices, PlaybackClock, Processing, ScheduledAudio, } from "./audio.js";
 export { type Client, type ClientOptions, createClient, DEFAULT_BASE_URL, type FactListParams, type Facts, type RelationListParams, type Relations, type SessionGetParams, type SessionListParams, type Sessions, } from "./client.js";
 export { type CloseInfo, type ConnectOptions, Conversation, type ConversationHandlers, type ConversationState, type Disconnect, type Dropped, READY_DEADLINE_MS, type Snapshot, type StartAudioOptions, type ToolHandler, type TranscriptOutcome, type TurnOptions, type TurnOutcome, type TurnSnapshot, } from "./conversation.js";
 export { AuthenticationError, ConflictError, ConnectionError, InsufficientCreditsError, NotFoundError, PermissionError, PlatformError, type PlatformErrorInit, RateLimitError, ServerError, TimeoutError, ValidationError, } from "./errors.js";

package/dist/web-audio/index.d.ts

@@ -9,7 +9,7 @@
  * button.onclick = () => audio.prime(); // in the click: browsers start audio only from one
  * ```
  */
-import { WebAudio } from "./web-audio.js";
-export type { WebAudio };
+import { WebAudio, type WebAudioOptions } from "./web-audio.js";
+export type { WebAudio, WebAudioOptions };
 /** A player and a recorder for one conversation. Call `prime()` from the click that starts it. */
-export declare function createWebAudio(): WebAudio;
+export declare function createWebAudio(options?: WebAudioOptions): WebAudio;

package/dist/web-audio/index.js

@@ -11,6 +11,6 @@
  */
 import { WebAudio } from "./web-audio.js";
 /** A player and a recorder for one conversation. Call `prime()` from the click that starts it. */
-export function createWebAudio() {
-    return new WebAudio();
+export function createWebAudio(options) {
+    return new WebAudio(options);
 }

package/dist/web-audio/player.d.ts

@@ -18,7 +18,15 @@
  *    in the past — otherwise it is scheduled to have already happened and is
  *    simply dropped, which is heard as a missing sentence.
  *  - **A tiny lead-in** (`SCHEDULE_AHEAD`) absorbs the jitter between a frame
- *    arriving and it being decoded, at the cost of a few milliseconds.
+ *    arriving and it being decoded, at the cost of a few milliseconds. An
+ *    app that reads each frame before it plays — a lipsync model — sets a
+ *    longer one (`leadIn`), and every reply's first word waits that long.
+ *
+ * **`push` answers when the frame starts**, on the context's clock, and the
+ * samples it queued: an app draws a face against them (`web-audio.ts`'s
+ * `onScheduled`). What is heard is that clock less the device's latency
+ * (`latency`), read live: Bluetooth, and a speaker chosen (`setSink`),
+ * change it.
  *
  * **The context runs at the device's own rate**; each buffer is made at the
  * stream's (24 kHz unless `audio.start` says otherwise) and the browser
@@ -37,6 +45,19 @@ export interface PcmPlayerOptions {
     channels?: number;
     /** Called when the queue drains — every scheduled source has finished. */
     onIdle?: () => void;
+    /** Seconds a run starts after it is pushed, once the queue ran dry: `SCHEDULE_AHEAD`. */
+    leadIn?: number;
+    /** The speaker the context plays through from its start (`setSink`). */
+    sinkId?: string;
+}
+/** A frame as `push` queued it. */
+export interface Pushed {
+    /** The whole samples queued: a split one joined, a trailing byte held back. */
+    pcm: ArrayBuffer;
+    sample_rate: number;
+    /** On the context's clock, in seconds. */
+    startsAt: number;
+    duration: number;
 }
 export declare class PcmPlayer {
     private context;
@@ -54,7 +75,24 @@ export declare class PcmPlayer {
     private volume;
     private readonly channels;
     private readonly onIdle?;
+    private readonly leadIn;
+    private sinkId?;
     constructor(options?: PcmPlayerOptions);
+    /** The context's clock, in seconds; `0` before `resume`. */
+    get now(): number;
+    /**
+     * Seconds from a sample leaving the context to it being heard: the
+     * browser's own processing and the device's. Read each time, since a
+     * Bluetooth headset or a new speaker changes it; `0` where a browser does
+     * not say (Safari has no `outputLatency`).
+     */
+    get latency(): number;
+    /**
+     * Play through the speaker `id` from now on, and on a context made later.
+     * Answers whether it took: a speaker unplugged since it was listed, or
+     * one the page may not use, leaves the one before.
+     */
+    setSink(id: string): Promise<boolean>;
     /**
      * The rate of the PCM pushed from here on: each `audio.start` carries its
      * window's own, since an agent's voice may be edited mid-conversation.
@@ -79,22 +117,23 @@ export declare class PcmPlayer {
     /** Whether anything is scheduled and still to play. */
     get playing(): boolean;
     /**
-     * Queue one blob of PCM.
+     * Queue one blob of PCM, and answer what was queued and when it starts —
+     * `null` when nothing was: a lone byte held for its other half.
      *
      * Ignored before {@link resume} — a caller that pushes audio into a player
      * nobody unlocked has a gesture problem, and buffering it would hide that
      * until the sound arrived minutes later in one burst.
      */
-    push(pcm: ArrayBuffer): void;
+    push(pcm: ArrayBuffer): Pushed | null;
     /**
      * Stop everything now — barge-in, or the turn being cancelled.
      *
      * Disconnecting the gain node rather than stopping each source: sources are
      * fire-and-forget once scheduled and holding references to them purely to
      * stop them is a leak waiting to happen. A fresh node is built on the next
-     * push.
+     * push. Answers whether audio still to play was dropped.
      */
-    stop(): void;
+    stop(): boolean;
     /** Release the audio device. Safe to call more than once. */
     close(): Promise<void>;
 }

package/dist/web-audio/player.js

@@ -18,7 +18,15 @@
  *    in the past — otherwise it is scheduled to have already happened and is
  *    simply dropped, which is heard as a missing sentence.
  *  - **A tiny lead-in** (`SCHEDULE_AHEAD`) absorbs the jitter between a frame
- *    arriving and it being decoded, at the cost of a few milliseconds.
+ *    arriving and it being decoded, at the cost of a few milliseconds. An
+ *    app that reads each frame before it plays — a lipsync model — sets a
+ *    longer one (`leadIn`), and every reply's first word waits that long.
+ *
+ * **`push` answers when the frame starts**, on the context's clock, and the
+ * samples it queued: an app draws a face against them (`web-audio.ts`'s
+ * `onScheduled`). What is heard is that clock less the device's latency
+ * (`latency`), read live: Bluetooth, and a speaker chosen (`setSink`),
+ * change it.
  *
  * **The context runs at the device's own rate**; each buffer is made at the
  * stream's (24 kHz unless `audio.start` says otherwise) and the browser
@@ -57,6 +65,40 @@ export class PcmPlayer {
         this.sampleRate = options.sampleRate ?? DEFAULT_SAMPLE_RATE;
         this.channels = options.channels ?? 1;
         this.onIdle = options.onIdle;
+        this.leadIn = options.leadIn ?? SCHEDULE_AHEAD;
+        this.sinkId = options.sinkId;
+    }
+    /** The context's clock, in seconds; `0` before `resume`. */
+    get now() {
+        return this.context?.currentTime ?? 0;
+    }
+    /**
+     * Seconds from a sample leaving the context to it being heard: the
+     * browser's own processing and the device's. Read each time, since a
+     * Bluetooth headset or a new speaker changes it; `0` where a browser does
+     * not say (Safari has no `outputLatency`).
+     */
+    get latency() {
+        const context = this.context;
+        return context
+            ? (context.outputLatency || 0) + (context.baseLatency || 0)
+            : 0;
+    }
+    /**
+     * Play through the speaker `id` from now on, and on a context made later.
+     * Answers whether it took: a speaker unplugged since it was listed, or
+     * one the page may not use, leaves the one before.
+     */
+    async setSink(id) {
+        try {
+            await this.context?.setSinkId(id);
+        }
+        catch {
+            // swallowed: the answer is `false`, and the speaker before plays on.
+            return false;
+        }
+        this.sinkId = id;
+        return true;
     }
     /**
      * The rate of the PCM pushed from here on: each `audio.start` carries its
@@ -83,6 +125,12 @@ export class PcmPlayer {
             this.gain = this.context.createGain();
             this.gain.gain.value = this.volume;
             this.gain.connect(this.context.destination);
+            // Not awaited: the click's `resume` below comes first.
+            if (this.sinkId) {
+                this.context.setSinkId(this.sinkId).catch(() => {
+                    // swallowed: gone since it was chosen — the default speaker plays.
+                });
+            }
         }
         if (this.context.state === "suspended")
             await this.context.resume();
@@ -102,7 +150,8 @@ export class PcmPlayer {
         return this.scheduled > 0;
     }
     /**
-     * Queue one blob of PCM.
+     * Queue one blob of PCM, and answer what was queued and when it starts —
+     * `null` when nothing was: a lone byte held for its other half.
      *
      * Ignored before {@link resume} — a caller that pushes audio into a player
      * nobody unlocked has a gesture problem, and buffering it would hide that
@@ -111,7 +160,7 @@ export class PcmPlayer {
     push(pcm) {
         const context = this.context;
         if (!context)
-            return;
+            return null;
         // **A split sample is carried, never dropped.** The edge sends whole
         // samples per frame (`apps/edge/src/voice/tts.ts` holds an odd byte for
         // the next blob), so this should never fire — but truncating instead is
@@ -129,7 +178,7 @@ export class PcmPlayer {
         const usable = bytes.length - (bytes.length % 2);
         this.remainder = bytes.slice(usable);
         if (usable < 2)
-            return;
+            return null;
         // Copied into its own buffer rather than viewed in place: `Int16Array`
         // needs a 2-byte-aligned offset, and a frame joined to a carried byte has
         // no such promise.
@@ -150,7 +199,7 @@ export class PcmPlayer {
         // "start before now", which the API resolves as "start immediately" for the
         // first blob and drops for every one after it.
         const now = context.currentTime;
-        const startAt = this.playhead > now ? this.playhead : now + SCHEDULE_AHEAD;
+        const startAt = this.playhead > now ? this.playhead : now + this.leadIn;
         source.start(startAt);
         this.playhead = startAt + buffer.duration;
         this.scheduled += 1;
@@ -162,6 +211,12 @@ export class PcmPlayer {
             if (this.scheduled === 0)
                 this.onIdle?.();
         };
+        return {
+            pcm: samples.buffer,
+            sample_rate: this.sampleRate,
+            startsAt: startAt,
+            duration: buffer.duration,
+        };
     }
     /**
      * Stop everything now — barge-in, or the turn being cancelled.
@@ -169,11 +224,11 @@ export class PcmPlayer {
      * Disconnecting the gain node rather than stopping each source: sources are
      * fire-and-forget once scheduled and holding references to them purely to
      * stop them is a leak waiting to happen. A fresh node is built on the next
-     * push.
+     * push. Answers whether audio still to play was dropped.
      */
     stop() {
         if (!this.context || !this.gain)
-            return;
+            return false;
         this.gain.disconnect();
         this.gain = this.context.createGain();
         this.gain.gain.value = this.volume;
@@ -186,6 +241,7 @@ export class PcmPlayer {
         this.remainder = new Uint8Array(0);
         if (wasPlaying)
             this.onIdle?.();
+        return wasPlaying;
     }
     /** Release the audio device. Safe to call more than once. */
     async close() {

package/dist/web-audio/recorder.d.ts

@@ -30,6 +30,7 @@
  * Loaded on the first press (`web-audio.ts`), so a page that never records
  * never parses it.
  */
+import type { Processing } from "../audio.js";
 /** The rate this records at, whatever is asked: the socket's default. */
 export declare const RECORD_SAMPLE_RATE = 24000;
 /** Samples per delivered chunk — about 85 ms at 24 kHz. Small enough that the
@@ -46,6 +47,10 @@ export interface PcmRecorderOptions {
     /** Called with each block of samples, already 16-bit little-endian. */
     onChunk: (pcm: ArrayBuffer) => void;
     sampleRate?: number;
+    /** The microphone's id (`enumerateDevices`); the system's default when unset or gone. */
+    input?: string;
+    /** The browser's cleanup: each on unless `false`. */
+    processing?: Processing;
 }
 /** Float samples to 16-bit signed little-endian, which is what the wire wants. */
 export declare function toPcm16(samples: Float32Array): ArrayBuffer;
@@ -72,6 +77,8 @@ export declare class PcmRecorder {
     private source;
     private readonly onChunk;
     private readonly sampleRate;
+    private readonly input?;
+    private readonly processing;
     /** Samples not yet delivered — see `FRAME_SAMPLES`. */
     private pending;
     private pendingSamples;

package/dist/web-audio/recorder.js

@@ -95,6 +95,8 @@ export class PcmRecorder {
         this.pendingSamples = 0;
         this.onChunk = options.onChunk;
         this.sampleRate = options.sampleRate ?? RECORD_SAMPLE_RATE;
+        this.input = options.input;
+        this.processing = options.processing ?? {};
     }
     /** True once `start()` has a live microphone and has not been stopped. */
     get recording() {
@@ -121,16 +123,30 @@ export class PcmRecorder {
     async start() {
         if (this.stream)
             return;
-        this.stream = await navigator.mediaDevices.getUserMedia({
-            audio: {
-                // The browser's own cleanup, which is better than anything worth doing
-                // here and costs nothing: a recogniser fed room echo transcribes it.
-                echoCancellation: true,
-                noiseSuppression: true,
-                autoGainControl: true,
-                channelCount: 1,
-            },
+        // The browser's own cleanup, which is better than anything worth doing
+        // here and costs nothing: a recogniser fed room echo transcribes it. An
+        // app may turn a part off — a music app its noise suppression.
+        const { echoCancellation, noiseSuppression, autoGainControl } = this.processing;
+        const audio = {
+            echoCancellation: echoCancellation !== false,
+            noiseSuppression: noiseSuppression !== false,
+            autoGainControl: autoGainControl !== false,
+            channelCount: 1,
+        };
+        const ask = (deviceId) => navigator.mediaDevices.getUserMedia({
+            audio: deviceId ? { ...audio, deviceId: { exact: deviceId } } : audio,
         });
+        try {
+            // `exact`, so the microphone chosen is the one that records...
+            this.stream = await ask(this.input);
+        }
+        catch (failure) {
+            // ...and one unplugged since it was chosen records from the default,
+            // rather than failing the press.
+            if (!this.input || failure.name !== "OverconstrainedError")
+                throw failure;
+            this.stream = await ask();
+        }
         try {
             this.context = new AudioContext({ sampleRate: this.sampleRate });
             // Autoplay policy suspends a context created outside a gesture. Pressing

package/dist/web-audio/web-audio.d.ts

@@ -18,25 +18,70 @@
  *   Edge and Firefox remove a tab's own sound from its microphone — Firefox
  *   only with the context at the device's rate, which the player keeps.
  *   Every browser on an iPhone or iPad is WebKit, whose canceller fails
- *   three ways (WebKit bugs 218012, 311451, 326286): refused there.
+ *   three ways (WebKit bugs 218012, 311451, 326286): refused there — and
+ *   wherever the app turned echo cancellation off. Half-duplex runs
+ *   anywhere (`listen`'s `halfDuplex`): it never asks.
+ * - **The clock and the frames are the app's** (`PlaybackClock`): each
+ *   frame told as it is queued, with when it starts; the stop; what the
+ *   microphone captures. A handler that throws is rethrown on its own turn,
+ *   never inside `play`, `stop` or a capture — a lipsync bug must not cut
+ *   the voice.
+ * - **The microphone, its processing and the speaker are the app's to
+ *   choose** (`DeviceChoices`): the speaker through
+ *   `AudioContext.setSinkId`, Chrome's and Edge's alone. Whether echo is
+ *   still cancelled with a speaker that is not the default is Chrome's,
+ *   and unchecked (left-overs #222).
  */
-import type { AgentAudio } from "../audio.js";
+import type { AgentAudio, AudioDevice, DeviceChoices, PlaybackClock, Processing, ScheduledAudio } from "../audio.js";
+export interface WebAudioOptions {
+    /**
+     * Milliseconds a reply's first frame — or one after a gap — waits before
+     * it plays: 60 unless set. Longer gives an app time to read each frame
+     * before it is heard (`onScheduled`), and adds as much to every first word.
+     */
+    leadInMs?: number;
+    /** The microphone, by its `inputs()` id: the default unless set. */
+    input?: string;
+    /** The browser's cleanup of the microphone: each on unless `false`. */
+    processing?: Processing;
+}
+/** What `play` is given: the frame's rate, and what `onScheduled` passes on when it has it. */
+type Frame = Parameters<AgentAudio["play"]>[1];
 /** Why hands-free is refused on an iPhone or iPad. */
-export declare const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use push-to-talk.";
+export declare const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use half-duplex or push-to-talk.";
 /** Why it is refused in a browser that cannot cancel echo at all. */
-export declare const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use push-to-talk.";
-export declare class WebAudio implements AgentAudio {
+export declare const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use half-duplex or push-to-talk.";
+/** Why it is refused with echo cancellation turned off (`setProcessing`). */
+export declare const ECHO_OFF = "Echo cancellation is turned off, so the agent would hear itself. Turn it on, or use half-duplex or push-to-talk.";
+export declare class WebAudio implements AgentAudio, PlaybackClock, DeviceChoices {
     private player;
     private recorder;
     private onPcm;
     private readonly drained;
+    private readonly scheduled;
+    private readonly stopped;
+    private readonly captured;
+    private readonly leadIn?;
+    private input?;
+    private processing;
+    private sink?;
+    constructor(options?: WebAudioOptions);
     /** Make the player and unlock it. **Call it from the click**, before any `await`. */
     prime(): Promise<void>;
-    play(pcm: ArrayBuffer, frame: {
-        sample_rate: number;
-    }): void;
+    play(pcm: ArrayBuffer, frame: Pick<Frame, "sample_rate"> & Partial<Frame>): void;
     stop(): void;
     onDrained(fn: () => void): () => void;
+    now(): number;
+    heardTime(): number;
+    onScheduled(fn: (audio: ScheduledAudio) => void): () => void;
+    onStopped(fn: () => void): () => void;
+    onCapture(fn: (pcm: ArrayBuffer, rate: number) => void): () => void;
+    inputs(): Promise<AudioDevice[]>;
+    setInput(id?: string): void;
+    setProcessing(processing: Processing): void;
+    outputs(): Promise<AudioDevice[]>;
+    setOutputDevice(id: string): Promise<boolean>;
+    get canChooseOutput(): boolean;
     /** At the recorder's own rate, 24 kHz, whatever is asked. */
     startCapture(onPcm: (pcm: ArrayBuffer) => void, _options?: {
         rate?: number;
@@ -54,3 +99,4 @@ export declare class WebAudio implements AgentAudio {
     /** The conversation is over: the player's context goes too. */
     dispose(): void;
 }
+export {};

package/dist/web-audio/web-audio.js

@@ -2,23 +2,31 @@ import { PcmPlayer } from "./player.js";
 /** The reply's loudness while the person may be talking over it: about -10 dB. */
 const DUCKED = 0.3;
 /** Why hands-free is refused on an iPhone or iPad. */
-export const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use push-to-talk.";
+export const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use half-duplex or push-to-talk.";
 /** Why it is refused in a browser that cannot cancel echo at all. */
-export const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use push-to-talk.";
+export const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use half-duplex or push-to-talk.";
+/** Why it is refused with echo cancellation turned off (`setProcessing`). */
+export const ECHO_OFF = "Echo cancellation is turned off, so the agent would hear itself. Turn it on, or use half-duplex or push-to-talk.";
 export class WebAudio {
-    constructor() {
+    constructor(options = {}) {
         this.player = null;
         this.recorder = null;
         this.onPcm = null;
         this.drained = new Set();
+        this.scheduled = new Set();
+        this.stopped = new Set();
+        this.captured = new Set();
+        if (options.leadInMs !== undefined)
+            this.leadIn = options.leadInMs / 1000;
+        this.input = options.input;
+        this.processing = options.processing ?? {};
     }
     /** Make the player and unlock it. **Call it from the click**, before any `await`. */
     prime() {
         this.player ?? (this.player = new PcmPlayer({
-            onIdle: () => {
-                for (const fn of this.drained)
-                    fn();
-            },
+            leadIn: this.leadIn,
+            sinkId: this.sink,
+            onIdle: () => tell(this.drained),
         }));
         return this.player.resume();
     }
@@ -27,21 +35,73 @@ export class WebAudio {
             void this.prime();
         const player = this.player;
         player.setSampleRate(frame.sample_rate);
-        player.push(pcm);
+        const queued = player.push(pcm);
+        if (queued) {
+            tell(this.scheduled, {
+                ...queued,
+                channels: frame.channels ?? 1,
+                chunk: frame.chunk ?? 0,
+                turn: frame.turn ?? "",
+            });
+        }
     }
     stop() {
-        this.player?.stop();
+        if (this.player?.stop())
+            tell(this.stopped);
     }
     onDrained(fn) {
-        this.drained.add(fn);
-        return () => {
-            this.drained.delete(fn);
-        };
+        return add(this.drained, fn);
+    }
+    now() {
+        return this.player?.now ?? 0;
+    }
+    heardTime() {
+        return Math.max(0, this.now() - (this.player?.latency ?? 0));
+    }
+    onScheduled(fn) {
+        return add(this.scheduled, fn);
+    }
+    onStopped(fn) {
+        return add(this.stopped, fn);
+    }
+    onCapture(fn) {
+        return add(this.captured, fn);
+    }
+    inputs() {
+        return devices("audioinput");
+    }
+    setInput(id) {
+        this.input = id;
+    }
+    setProcessing(processing) {
+        this.processing = processing;
+    }
+    async outputs() {
+        return this.canChooseOutput ? devices("audiooutput") : [];
+    }
+    async setOutputDevice(id) {
+        if (!this.canChooseOutput)
+            return false;
+        if (this.player && !(await this.player.setSink(id)))
+            return false;
+        this.sink = id;
+        return true;
+    }
+    get canChooseOutput() {
+        return (typeof AudioContext !== "undefined" &&
+            "setSinkId" in AudioContext.prototype);
     }
     /** At the recorder's own rate, 24 kHz, whatever is asked. */
     async startCapture(onPcm, _options) {
         const { PcmRecorder } = await import("./recorder.js");
-        const mic = new PcmRecorder({ onChunk: (pcm) => this.onPcm?.(pcm) });
+        const mic = new PcmRecorder({
+            input: this.input,
+            processing: this.processing,
+            onChunk: (pcm) => {
+                this.onPcm?.(pcm);
+                tell(this.captured, pcm, mic.rate);
+            },
+        });
         this.onPcm = onPcm;
         this.recorder = mic;
         try {
@@ -59,16 +119,24 @@ export class WebAudio {
         const deliver = this.onPcm;
         this.recorder = null;
         this.onPcm = null;
-        const tail = mic?.stop();
-        if (tail)
-            deliver?.(tail);
+        if (!mic)
+            return;
+        const rate = mic.rate;
+        const tail = mic.stop();
+        if (!tail)
+            return;
+        deliver?.(tail);
+        tell(this.captured, tail, rate);
     }
     duck(on) {
         this.player?.setVolume(on ? DUCKED : 1);
     }
     async handsFree() {
-        const devices = typeof navigator === "undefined" ? undefined : navigator.mediaDevices;
-        if (!devices?.getSupportedConstraints?.().echoCancellation) {
+        if (this.processing.echoCancellation === false) {
+            return { ok: false, reason: ECHO_OFF };
+        }
+        const media = mediaDevices();
+        if (!media?.getSupportedConstraints?.().echoCancellation) {
             return { ok: false, reason: NO_ECHO };
         }
         // iPadOS says it is a Mac: a Mac with a touch screen is an iPad.
@@ -83,3 +151,34 @@ export class WebAudio {
         this.player = null;
     }
 }
+const mediaDevices = () => typeof navigator === "undefined" ? undefined : navigator.mediaDevices;
+/** The browser's devices of `kind`; ids and labels empty until the microphone was allowed. */
+async function devices(kind) {
+    const listed = (await mediaDevices()?.enumerateDevices?.()) ?? [];
+    return listed
+        .filter((device) => device.kind === kind)
+        .map(({ deviceId, label }) => ({ id: deviceId, label }));
+}
+function add(listeners, fn) {
+    listeners.add(fn);
+    return () => {
+        listeners.delete(fn);
+    };
+}
+/**
+ * Call each of the app's handlers. One that throws is rethrown on its own
+ * turn of the event loop, where the app's error reporting sees it — not
+ * here, where it would stop the player or the recorder halfway.
+ */
+function tell(listeners, ...args) {
+    for (const fn of listeners) {
+        try {
+            fn(...args);
+        }
+        catch (error) {
+            setTimeout(() => {
+                throw error;
+            });
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lumavox/client",
-  "version": "0.15.1",
+  "version": "0.17.0",
   "description": "Talk to Lumavox agents from a browser, a React Native app or Node, with an end user's ephemeral key: conversations on the agent's socket, sessions, memory and relations. Zero dependencies.",
   "license": "MIT",
   "type": "module",