@lumavox/client 0.14.1 → 0.16.0

package/README.md

@@ -91,7 +91,7 @@ key, dials, authenticates, and from then on:
 
 | Handler | Frame |
 |---|---|
-| `onReady` `onTurnStart` `onHeard` `onHeardPartial` `onKnowledge` `onMood` `onThought` `onText` `onChunk` `onToolStart` `onToolEnd` `onToolConfirm` `onUsage` `onMuted` `onAudioStart` `onAudioEnd` `onTranscript` `onDone` `onInterrupted` `onError` | the wire's frame, as it is |
+| `onReady` `onTurnStart` `onHeard` `onHeardPartial` `onKnowledge` `onMood` `onThought` `onText` `onChunk` `onToolStart` `onToolEnd` `onToolConfirm` `onUsage` `onMuted` `onAudioStart` `onAudioEnd` `onTranscript` `onCompacted` `onDone` `onInterrupted` `onError` | the wire's frame, as it is |
 | `onAudio` | `{ pcm, chunk, turn, sample_rate, channels }`: raw 16-bit PCM, never copied |
 | `onRelation` | the relationship, each change once |
 | `onState` | `connecting` `ready` `busy` `reconnecting` `dormant` `closed` |
@@ -143,12 +143,37 @@ startButton.onclick = () => audio.prime(); // in the click: browsers start sound
 ```
 
 An `AgentAudio` on Web Audio, its own import so the client carries none of
-it (~5.5 KB minified): replies play gaplessly as they arrive, at the
+it (~8.5 KB minified): replies play gaplessly as they arrive, at the
 device's own rate; the microphone records at 24 kHz, with the browser's
 echo cancellation, noise suppression and gain control. Hand it to
 `@lumavox/react`'s `useAgent`, or call `play`, `startCapture` and the rest
 yourself.
 
+**The clock**, to draw a face or subtitles against the voice:
+
+```ts
+const audio = createWebAudio({ leadInMs: 350 }); // time to read each frame before it plays
+audio.onScheduled(({ pcm, sample_rate, startsAt }) => lipsync.add(pcm, sample_rate, startsAt));
+audio.onStopped(() => lipsync.clear()); // a cancel, a mute, a barge-in
+// each animation frame: lipsync.at(audio.heardTime())
+```
+
+- `onScheduled`: every frame as it is queued, its 16-bit samples and when
+  they start on `now()`'s clock. `heardTime()`: what is heard at this
+  moment, the clock less the browser's output latency (Bluetooth adds to
+  it).
+- `leadInMs`: a reply's first frame, or one after a gap, waits that long —
+  60 ms unless set; it adds as much to the first word.
+- `onStopped`: a `stop()` that dropped audio still to play. `onCapture((pcm,
+  rate) => …)`: every frame the microphone captures, for a level meter.
+
+**The devices**: `inputs()` and `setInput(id)` for the microphone (one
+unplugged falls back to the default; labels are empty until it was
+allowed), `setProcessing({ echoCancellation, noiseSuppression,
+autoGainControl })` — each on unless `false`, and hands-free refused with
+echo cancellation off — and, where `canChooseOutput` (Chrome and Edge),
+`outputs()` and `setOutputDevice(id)` for the speaker.
+
 ### Hands-free
 
 ```ts

package/dist/audio.d.ts

@@ -8,7 +8,11 @@
  * mute or a new turn, records at the rate the recorder was granted, and
  * throws away a recording the device interrupted. Hands-free — the
  * microphone left open, a detector saying when the person talks — is
- * `listen` (`./hands-free.ts`), on the two optional methods below.
+ * `listen` (`./hands-free.ts`), on the two optional methods below. The
+ * clock, the frames as they are queued and captured, and the devices
+ * (`PlaybackClock`, `DeviceChoices`) are the two adapters' own, for the
+ * app: nothing that drives an `AgentAudio` needs them, so an adapter of an
+ * app's own need not have them.
  */
 export interface AgentAudio {
     /** Queue one frame of a reply: 16-bit little-endian PCM, in wire order. */
@@ -62,3 +66,86 @@ export interface AgentAudio {
     /** Optional: play the reply quieter, or as before — barge-in's first step, on sound alone. */
     duck?(on: boolean): void;
 }
+/** One frame of a reply as the player queued it: what a face or subtitles are drawn against. */
+export interface ScheduledAudio {
+    /** The 16-bit little-endian samples queued: a sample split across two frames already joined. */
+    pcm: ArrayBuffer;
+    sample_rate: number;
+    channels: number;
+    /** As the socket sent it: the text chunk this is the voice of, in its turn. */
+    chunk: number;
+    turn: string;
+    /** When it starts to play, in seconds on `now()`'s clock. */
+    startsAt: number;
+    /** Seconds. */
+    duration: number;
+}
+/**
+ * The player's clock and what passes through it, for an app that draws
+ * against the voice — a face lip-synced, subtitles revealed chunk by chunk
+ * — or against the microphone, a level meter. `@lumavox/client/web-audio`'s
+ * and `@lumavox/react-native`'s audio have it; `useAgent` needs none of it.
+ */
+export interface PlaybackClock {
+    /** The player's clock, in seconds: its audio context's. `0` before `prime()`. */
+    now(): number;
+    /**
+     * What the listener hears at this moment, on `now()`'s clock: `now()`
+     * less the time the sound takes to leave the device, which Bluetooth
+     * makes longer. Never negative.
+     */
+    heardTime(): number;
+    /**
+     * Every frame as it is queued, before it plays: its samples and when
+     * they start. A handler that throws is rethrown on its own. Answers the
+     * unsubscribe.
+     */
+    onScheduled(fn: (audio: ScheduledAudio) => void): () => void;
+    /**
+     * When `stop()` drops audio still to play — a cancel, a mute, a
+     * barge-in, a new turn. `onDrained` fires then too, and when a reply
+     * plays out. Answers the unsubscribe.
+     */
+    onStopped(fn: () => void): () => void;
+    /**
+     * Every frame the microphone captures while it is open — what
+     * `startCapture` hands on, and what hands-free does not send — as 16-bit
+     * PCM at `rate`. Answers the unsubscribe.
+     */
+    onCapture(fn: (pcm: ArrayBuffer, rate: number) => void): () => void;
+}
+/** A microphone or a speaker, as the browser lists it. */
+export interface AudioDevice {
+    id: string;
+    /** Empty until the person has allowed the microphone. */
+    label: string;
+}
+/** The browser's own cleanup of the microphone: each on unless `false`. */
+export interface Processing {
+    /** Off, hands-free is refused: the agent would hear itself. */
+    echoCancellation?: boolean;
+    noiseSuppression?: boolean;
+    autoGainControl?: boolean;
+}
+/**
+ * The microphone and the speaker a browser lets a page choose. On a phone
+ * the system chooses: no devices are listed and none can be set.
+ */
+export interface DeviceChoices {
+    /** The microphones; labels are empty until the microphone was allowed. */
+    inputs(): Promise<AudioDevice[]>;
+    /** The microphone, by its id, from the next capture; none, the default. One unplugged since falls back to the default. */
+    setInput(id?: string): void;
+    /** The browser's processing from the next capture: what is left out is on. */
+    setProcessing(processing: Processing): void;
+    /** The speakers a page can play through; `[]` where it cannot choose. */
+    outputs(): Promise<AudioDevice[]>;
+    /**
+     * Play through the speaker `id`, kept for a player made later. Answers
+     * whether it took: `false` where the browser cannot choose, or the
+     * speaker is gone.
+     */
+    setOutputDevice(id: string): Promise<boolean>;
+    /** Whether this browser lets a page choose its speaker: Chrome and Edge. */
+    readonly canChooseOutput: boolean;
+}

package/dist/conversation.d.ts

@@ -157,6 +157,8 @@ export interface ConversationHandlers {
     onInterrupted?: On<"interrupted">;
     /** The relationship, from `ready` and when it moves: each change once. */
     onRelation?: (relation: RelationInfo) => void;
+    /** The conversation was compacted, after a reply: its transcript is unchanged. */
+    onCompacted?: On<"compacted">;
     onError?: On<"error">;
     onState?: (state: ConversationState) => void;
     /** Only when it will **not** reconnect. */

package/dist/conversation.js

@@ -33,6 +33,7 @@ const HANDLER = {
     "audio.start": "onAudioStart",
     "audio.end": "onAudioEnd",
     transcript: "onTranscript",
+    compacted: "onCompacted",
     "turn.done": "onDone",
     interrupted: "onInterrupted",
     error: "onError",

package/dist/events.d.ts

@@ -182,6 +182,18 @@ export interface MutedFrame extends Frame<"muted">, InTurn {
 /** The relationship moved on this turn: its new fields. */
 export interface RelationFrame extends Frame<"relation">, InTurn, RelationInfo {
 }
+/**
+ * The conversation was compacted — the agent's own past its threshold, a
+ * pass of a long one, or one asked for — after a reply, never in one: its
+ * next turn starts from a summary through `after_seq`, then the messages
+ * after it word for word. Not part of any turn.
+ */
+export interface CompactedFrame extends Frame<"compacted"> {
+    session_id: string;
+    /** The last message the summary covers. */
+    after_seq: number;
+    trigger: "automatic" | "manual";
+}
 export interface UsageFrame extends Frame<"usage">, InTurn {
     model: string;
     input_tokens: number;
@@ -282,6 +294,7 @@ export type ServerFrames = Exhaustive<{
     "audio.end": AudioEndFrame;
     muted: MutedFrame;
     relation: RelationFrame;
+    compacted: CompactedFrame;
     usage: UsageFrame;
     "turn.done": TurnDoneFrame;
     interrupted: InterruptedFrame;

package/dist/gen/types.gen.d.ts

@@ -270,7 +270,7 @@ export type SessionInfo = {
 };
 export type CompactionInfo = {
     /**
-     * The last message this summary covers. Anything after it is still sent to the model verbatim.
+     * The last message this summary covers. Anything after it — the last few exchanges, kept word for word, then whatever is said next — is still sent to the model verbatim.
      */
     after_seq: number;
     /**
@@ -289,7 +289,7 @@ export type CompactionInfo = {
      */
     summary: string;
     /**
-     * `manual` (a client asked), `automatic` (a turn reached the agent's threshold) or `idle` (the conversation went quiet and its facts were extracted).
+     * `manual` (a client asked) or `automatic` (a turn reached the agent's threshold, or a long compaction's next pass). `idle` is no longer written: it marks compactions made when a quiet conversation's facts were read, before facts were read on their own.
      */
     trigger: 'manual' | 'automatic' | 'idle';
 };
@@ -553,6 +553,10 @@ export type CompactSessionErrors = {
      * Missing or invalid credentials
      */
     401: ErrorResponse;
+    /**
+     * The key has spent its credit — mint another; nothing was charged
+     */
+    402: ErrorResponse;
     /**
      * The key is valid but its scopes do not cover this capability
      */
@@ -566,7 +570,7 @@ export type CompactSessionErrors = {
      */
     409: ErrorResponse;
     /**
-     * The request failed validation
+     * Nothing new to compact since the last compaction, or the model's summary was empty or no shorter than what it would replace; nothing was charged
      */
     422: ErrorResponse;
     /**
@@ -603,6 +607,10 @@ export type ExtractSessionFactsErrors = {
      * Missing or invalid credentials
      */
     401: ErrorResponse;
+    /**
+     * The key has spent its credit — mint another; nothing was charged
+     */
+    402: ErrorResponse;
     /**
      * The key is valid but its scopes do not cover this capability
      */
@@ -616,7 +624,7 @@ export type ExtractSessionFactsErrors = {
      */
     409: ErrorResponse;
     /**
-     * The request failed validation
+     * Nothing new to compact since the last compaction, or the model's summary was empty or no shorter than what it would replace; nothing was charged
      */
     422: ErrorResponse;
     /**

package/dist/index.d.ts

@@ -7,7 +7,7 @@
  * vocabulary (`src/protocol.gen.ts`); methods are camelCase, and the SDK's
  * own options go last.
  */
-export type { AgentAudio } from "./audio.js";
+export type { AgentAudio, AudioDevice, DeviceChoices, PlaybackClock, Processing, ScheduledAudio, } from "./audio.js";
 export { type Client, type ClientOptions, createClient, DEFAULT_BASE_URL, type FactListParams, type Facts, type RelationListParams, type Relations, type SessionGetParams, type SessionListParams, type Sessions, } from "./client.js";
 export { type CloseInfo, type ConnectOptions, Conversation, type ConversationHandlers, type ConversationState, type Disconnect, type Dropped, READY_DEADLINE_MS, type Snapshot, type StartAudioOptions, type ToolHandler, type TranscriptOutcome, type TurnOptions, type TurnOutcome, type TurnSnapshot, } from "./conversation.js";
 export { AuthenticationError, ConflictError, ConnectionError, InsufficientCreditsError, NotFoundError, PermissionError, PlatformError, type PlatformErrorInit, RateLimitError, ServerError, TimeoutError, ValidationError, } from "./errors.js";

package/dist/protocol.gen.d.ts

@@ -4,7 +4,7 @@ export declare const PROTOCOL = "ino.v2";
 export declare const CLIENT_VERBS: readonly ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping", "tool.confirmed", "tool.result"];
 export type ClientVerb = (typeof CLIENT_VERBS)[number];
 /** Every event the edge sends. */
-export declare const SERVER_EVENTS: readonly ["ready", "turn.start", "heard", "heard.partial", "barge_in", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
+export declare const SERVER_EVENTS: readonly ["ready", "turn.start", "heard", "heard.partial", "barge_in", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "compacted", "usage", "turn.done", "interrupted", "error", "pong"];
 export type ServerEventName = (typeof SERVER_EVENTS)[number];
 /** The events that end a turn: exactly one of them ends every turn. */
 export declare const TERMINAL_EVENTS: readonly ["turn.done", "interrupted"];

package/dist/protocol.gen.js

@@ -7,7 +7,7 @@ export const PROTOCOL = "ino.v2";
 /** Every frame a client may send; the edge closes on any other (4009). */
 export const CLIENT_VERBS = ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping", "tool.confirmed", "tool.result"];
 /** Every event the edge sends. */
-export const SERVER_EVENTS = ["ready", "turn.start", "heard", "heard.partial", "barge_in", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
+export const SERVER_EVENTS = ["ready", "turn.start", "heard", "heard.partial", "barge_in", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "compacted", "usage", "turn.done", "interrupted", "error", "pong"];
 /** The events that end a turn: exactly one of them ends every turn. */
 export const TERMINAL_EVENTS = ["turn.done", "interrupted"];
 /** A `ping` of exactly this text is answered by the edge's runtime with `KEEPALIVE_PONG`, which has no `seq`. */

package/dist/web-audio/index.d.ts

@@ -9,7 +9,7 @@
  * button.onclick = () => audio.prime(); // in the click: browsers start audio only from one
  * ```
  */
-import { WebAudio } from "./web-audio.js";
-export type { WebAudio };
+import { WebAudio, type WebAudioOptions } from "./web-audio.js";
+export type { WebAudio, WebAudioOptions };
 /** A player and a recorder for one conversation. Call `prime()` from the click that starts it. */
-export declare function createWebAudio(): WebAudio;
+export declare function createWebAudio(options?: WebAudioOptions): WebAudio;

package/dist/web-audio/index.js

@@ -11,6 +11,6 @@
  */
 import { WebAudio } from "./web-audio.js";
 /** A player and a recorder for one conversation. Call `prime()` from the click that starts it. */
-export function createWebAudio() {
-    return new WebAudio();
+export function createWebAudio(options) {
+    return new WebAudio(options);
 }

package/dist/web-audio/player.d.ts

@@ -18,7 +18,15 @@
  *    in the past — otherwise it is scheduled to have already happened and is
  *    simply dropped, which is heard as a missing sentence.
  *  - **A tiny lead-in** (`SCHEDULE_AHEAD`) absorbs the jitter between a frame
- *    arriving and it being decoded, at the cost of a few milliseconds.
+ *    arriving and it being decoded, at the cost of a few milliseconds. An
+ *    app that reads each frame before it plays — a lipsync model — sets a
+ *    longer one (`leadIn`), and every reply's first word waits that long.
+ *
+ * **`push` answers when the frame starts**, on the context's clock, and the
+ * samples it queued: an app draws a face against them (`web-audio.ts`'s
+ * `onScheduled`). What is heard is that clock less the device's latency
+ * (`latency`), read live: Bluetooth, and a speaker chosen (`setSink`),
+ * change it.
  *
  * **The context runs at the device's own rate**; each buffer is made at the
  * stream's (24 kHz unless `audio.start` says otherwise) and the browser
@@ -37,6 +45,19 @@ export interface PcmPlayerOptions {
     channels?: number;
     /** Called when the queue drains — every scheduled source has finished. */
     onIdle?: () => void;
+    /** Seconds a run starts after it is pushed, once the queue ran dry: `SCHEDULE_AHEAD`. */
+    leadIn?: number;
+    /** The speaker the context plays through from its start (`setSink`). */
+    sinkId?: string;
+}
+/** A frame as `push` queued it. */
+export interface Pushed {
+    /** The whole samples queued: a split one joined, a trailing byte held back. */
+    pcm: ArrayBuffer;
+    sample_rate: number;
+    /** On the context's clock, in seconds. */
+    startsAt: number;
+    duration: number;
 }
 export declare class PcmPlayer {
     private context;
@@ -54,7 +75,24 @@ export declare class PcmPlayer {
     private volume;
     private readonly channels;
     private readonly onIdle?;
+    private readonly leadIn;
+    private sinkId?;
     constructor(options?: PcmPlayerOptions);
+    /** The context's clock, in seconds; `0` before `resume`. */
+    get now(): number;
+    /**
+     * Seconds from a sample leaving the context to it being heard: the
+     * browser's own processing and the device's. Read each time, since a
+     * Bluetooth headset or a new speaker changes it; `0` where a browser does
+     * not say (Safari has no `outputLatency`).
+     */
+    get latency(): number;
+    /**
+     * Play through the speaker `id` from now on, and on a context made later.
+     * Answers whether it took: a speaker unplugged since it was listed, or
+     * one the page may not use, leaves the one before.
+     */
+    setSink(id: string): Promise<boolean>;
     /**
      * The rate of the PCM pushed from here on: each `audio.start` carries its
      * window's own, since an agent's voice may be edited mid-conversation.
@@ -79,22 +117,23 @@ export declare class PcmPlayer {
     /** Whether anything is scheduled and still to play. */
     get playing(): boolean;
     /**
-     * Queue one blob of PCM.
+     * Queue one blob of PCM, and answer what was queued and when it starts —
+     * `null` when nothing was: a lone byte held for its other half.
      *
      * Ignored before {@link resume} — a caller that pushes audio into a player
      * nobody unlocked has a gesture problem, and buffering it would hide that
      * until the sound arrived minutes later in one burst.
      */
-    push(pcm: ArrayBuffer): void;
+    push(pcm: ArrayBuffer): Pushed | null;
     /**
      * Stop everything now — barge-in, or the turn being cancelled.
      *
      * Disconnecting the gain node rather than stopping each source: sources are
      * fire-and-forget once scheduled and holding references to them purely to
      * stop them is a leak waiting to happen. A fresh node is built on the next
-     * push.
+     * push. Answers whether audio still to play was dropped.
      */
-    stop(): void;
+    stop(): boolean;
     /** Release the audio device. Safe to call more than once. */
     close(): Promise<void>;
 }

package/dist/web-audio/player.js

@@ -18,7 +18,15 @@
  *    in the past — otherwise it is scheduled to have already happened and is
  *    simply dropped, which is heard as a missing sentence.
  *  - **A tiny lead-in** (`SCHEDULE_AHEAD`) absorbs the jitter between a frame
- *    arriving and it being decoded, at the cost of a few milliseconds.
+ *    arriving and it being decoded, at the cost of a few milliseconds. An
+ *    app that reads each frame before it plays — a lipsync model — sets a
+ *    longer one (`leadIn`), and every reply's first word waits that long.
+ *
+ * **`push` answers when the frame starts**, on the context's clock, and the
+ * samples it queued: an app draws a face against them (`web-audio.ts`'s
+ * `onScheduled`). What is heard is that clock less the device's latency
+ * (`latency`), read live: Bluetooth, and a speaker chosen (`setSink`),
+ * change it.
  *
  * **The context runs at the device's own rate**; each buffer is made at the
  * stream's (24 kHz unless `audio.start` says otherwise) and the browser
@@ -57,6 +65,40 @@ export class PcmPlayer {
         this.sampleRate = options.sampleRate ?? DEFAULT_SAMPLE_RATE;
         this.channels = options.channels ?? 1;
         this.onIdle = options.onIdle;
+        this.leadIn = options.leadIn ?? SCHEDULE_AHEAD;
+        this.sinkId = options.sinkId;
+    }
+    /** The context's clock, in seconds; `0` before `resume`. */
+    get now() {
+        return this.context?.currentTime ?? 0;
+    }
+    /**
+     * Seconds from a sample leaving the context to it being heard: the
+     * browser's own processing and the device's. Read each time, since a
+     * Bluetooth headset or a new speaker changes it; `0` where a browser does
+     * not say (Safari has no `outputLatency`).
+     */
+    get latency() {
+        const context = this.context;
+        return context
+            ? (context.outputLatency || 0) + (context.baseLatency || 0)
+            : 0;
+    }
+    /**
+     * Play through the speaker `id` from now on, and on a context made later.
+     * Answers whether it took: a speaker unplugged since it was listed, or
+     * one the page may not use, leaves the one before.
+     */
+    async setSink(id) {
+        try {
+            await this.context?.setSinkId(id);
+        }
+        catch {
+            // swallowed: the answer is `false`, and the speaker before plays on.
+            return false;
+        }
+        this.sinkId = id;
+        return true;
     }
     /**
      * The rate of the PCM pushed from here on: each `audio.start` carries its
@@ -83,6 +125,12 @@ export class PcmPlayer {
             this.gain = this.context.createGain();
             this.gain.gain.value = this.volume;
             this.gain.connect(this.context.destination);
+            // Not awaited: the click's `resume` below comes first.
+            if (this.sinkId) {
+                this.context.setSinkId(this.sinkId).catch(() => {
+                    // swallowed: gone since it was chosen — the default speaker plays.
+                });
+            }
         }
         if (this.context.state === "suspended")
             await this.context.resume();
@@ -102,7 +150,8 @@ export class PcmPlayer {
         return this.scheduled > 0;
     }
     /**
-     * Queue one blob of PCM.
+     * Queue one blob of PCM, and answer what was queued and when it starts —
+     * `null` when nothing was: a lone byte held for its other half.
      *
      * Ignored before {@link resume} — a caller that pushes audio into a player
      * nobody unlocked has a gesture problem, and buffering it would hide that
@@ -111,7 +160,7 @@ export class PcmPlayer {
     push(pcm) {
         const context = this.context;
         if (!context)
-            return;
+            return null;
         // **A split sample is carried, never dropped.** The edge sends whole
         // samples per frame (`apps/edge/src/voice/tts.ts` holds an odd byte for
         // the next blob), so this should never fire — but truncating instead is
@@ -129,7 +178,7 @@ export class PcmPlayer {
         const usable = bytes.length - (bytes.length % 2);
         this.remainder = bytes.slice(usable);
         if (usable < 2)
-            return;
+            return null;
         // Copied into its own buffer rather than viewed in place: `Int16Array`
         // needs a 2-byte-aligned offset, and a frame joined to a carried byte has
         // no such promise.
@@ -150,7 +199,7 @@ export class PcmPlayer {
         // "start before now", which the API resolves as "start immediately" for the
         // first blob and drops for every one after it.
         const now = context.currentTime;
-        const startAt = this.playhead > now ? this.playhead : now + SCHEDULE_AHEAD;
+        const startAt = this.playhead > now ? this.playhead : now + this.leadIn;
         source.start(startAt);
         this.playhead = startAt + buffer.duration;
         this.scheduled += 1;
@@ -162,6 +211,12 @@ export class PcmPlayer {
             if (this.scheduled === 0)
                 this.onIdle?.();
         };
+        return {
+            pcm: samples.buffer,
+            sample_rate: this.sampleRate,
+            startsAt: startAt,
+            duration: buffer.duration,
+        };
     }
     /**
      * Stop everything now — barge-in, or the turn being cancelled.
@@ -169,11 +224,11 @@ export class PcmPlayer {
      * Disconnecting the gain node rather than stopping each source: sources are
      * fire-and-forget once scheduled and holding references to them purely to
      * stop them is a leak waiting to happen. A fresh node is built on the next
-     * push.
+     * push. Answers whether audio still to play was dropped.
      */
     stop() {
         if (!this.context || !this.gain)
-            return;
+            return false;
         this.gain.disconnect();
         this.gain = this.context.createGain();
         this.gain.gain.value = this.volume;
@@ -186,6 +241,7 @@ export class PcmPlayer {
         this.remainder = new Uint8Array(0);
         if (wasPlaying)
             this.onIdle?.();
+        return wasPlaying;
     }
     /** Release the audio device. Safe to call more than once. */
     async close() {

package/dist/web-audio/recorder.d.ts

@@ -30,6 +30,7 @@
  * Loaded on the first press (`web-audio.ts`), so a page that never records
  * never parses it.
  */
+import type { Processing } from "../audio.js";
 /** The rate this records at, whatever is asked: the socket's default. */
 export declare const RECORD_SAMPLE_RATE = 24000;
 /** Samples per delivered chunk — about 85 ms at 24 kHz. Small enough that the
@@ -46,6 +47,10 @@ export interface PcmRecorderOptions {
     /** Called with each block of samples, already 16-bit little-endian. */
     onChunk: (pcm: ArrayBuffer) => void;
     sampleRate?: number;
+    /** The microphone's id (`enumerateDevices`); the system's default when unset or gone. */
+    input?: string;
+    /** The browser's cleanup: each on unless `false`. */
+    processing?: Processing;
 }
 /** Float samples to 16-bit signed little-endian, which is what the wire wants. */
 export declare function toPcm16(samples: Float32Array): ArrayBuffer;
@@ -72,6 +77,8 @@ export declare class PcmRecorder {
     private source;
     private readonly onChunk;
     private readonly sampleRate;
+    private readonly input?;
+    private readonly processing;
     /** Samples not yet delivered — see `FRAME_SAMPLES`. */
     private pending;
     private pendingSamples;

package/dist/web-audio/recorder.js

@@ -95,6 +95,8 @@ export class PcmRecorder {
         this.pendingSamples = 0;
         this.onChunk = options.onChunk;
         this.sampleRate = options.sampleRate ?? RECORD_SAMPLE_RATE;
+        this.input = options.input;
+        this.processing = options.processing ?? {};
     }
     /** True once `start()` has a live microphone and has not been stopped. */
     get recording() {
@@ -121,16 +123,30 @@ export class PcmRecorder {
     async start() {
         if (this.stream)
             return;
-        this.stream = await navigator.mediaDevices.getUserMedia({
-            audio: {
-                // The browser's own cleanup, which is better than anything worth doing
-                // here and costs nothing: a recogniser fed room echo transcribes it.
-                echoCancellation: true,
-                noiseSuppression: true,
-                autoGainControl: true,
-                channelCount: 1,
-            },
+        // The browser's own cleanup, which is better than anything worth doing
+        // here and costs nothing: a recogniser fed room echo transcribes it. An
+        // app may turn a part off — a music app its noise suppression.
+        const { echoCancellation, noiseSuppression, autoGainControl } = this.processing;
+        const audio = {
+            echoCancellation: echoCancellation !== false,
+            noiseSuppression: noiseSuppression !== false,
+            autoGainControl: autoGainControl !== false,
+            channelCount: 1,
+        };
+        const ask = (deviceId) => navigator.mediaDevices.getUserMedia({
+            audio: deviceId ? { ...audio, deviceId: { exact: deviceId } } : audio,
         });
+        try {
+            // `exact`, so the microphone chosen is the one that records...
+            this.stream = await ask(this.input);
+        }
+        catch (failure) {
+            // ...and one unplugged since it was chosen records from the default,
+            // rather than failing the press.
+            if (!this.input || failure.name !== "OverconstrainedError")
+                throw failure;
+            this.stream = await ask();
+        }
         try {
             this.context = new AudioContext({ sampleRate: this.sampleRate });
             // Autoplay policy suspends a context created outside a gesture. Pressing

package/dist/web-audio/web-audio.d.ts

@@ -18,25 +18,69 @@
  *   Edge and Firefox remove a tab's own sound from its microphone — Firefox
  *   only with the context at the device's rate, which the player keeps.
  *   Every browser on an iPhone or iPad is WebKit, whose canceller fails
- *   three ways (WebKit bugs 218012, 311451, 326286): refused there.
+ *   three ways (WebKit bugs 218012, 311451, 326286): refused there — and
+ *   wherever the app turned echo cancellation off.
+ * - **The clock and the frames are the app's** (`PlaybackClock`): each
+ *   frame told as it is queued, with when it starts; the stop; what the
+ *   microphone captures. A handler that throws is rethrown on its own turn,
+ *   never inside `play`, `stop` or a capture — a lipsync bug must not cut
+ *   the voice.
+ * - **The microphone, its processing and the speaker are the app's to
+ *   choose** (`DeviceChoices`): the speaker through
+ *   `AudioContext.setSinkId`, Chrome's and Edge's alone. Whether echo is
+ *   still cancelled with a speaker that is not the default is Chrome's,
+ *   and unchecked (left-overs #222).
  */
-import type { AgentAudio } from "../audio.js";
+import type { AgentAudio, AudioDevice, DeviceChoices, PlaybackClock, Processing, ScheduledAudio } from "../audio.js";
+export interface WebAudioOptions {
+    /**
+     * Milliseconds a reply's first frame — or one after a gap — waits before
+     * it plays: 60 unless set. Longer gives an app time to read each frame
+     * before it is heard (`onScheduled`), and adds as much to every first word.
+     */
+    leadInMs?: number;
+    /** The microphone, by its `inputs()` id: the default unless set. */
+    input?: string;
+    /** The browser's cleanup of the microphone: each on unless `false`. */
+    processing?: Processing;
+}
+/** What `play` is given: the frame's rate, and what `onScheduled` passes on when it has it. */
+type Frame = Parameters<AgentAudio["play"]>[1];
 /** Why hands-free is refused on an iPhone or iPad. */
 export declare const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use push-to-talk.";
 /** Why it is refused in a browser that cannot cancel echo at all. */
 export declare const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use push-to-talk.";
-export declare class WebAudio implements AgentAudio {
+/** Why it is refused with echo cancellation turned off (`setProcessing`). */
+export declare const ECHO_OFF = "Echo cancellation is turned off, so the agent would hear itself. Turn it on, or use push-to-talk.";
+export declare class WebAudio implements AgentAudio, PlaybackClock, DeviceChoices {
     private player;
     private recorder;
     private onPcm;
     private readonly drained;
+    private readonly scheduled;
+    private readonly stopped;
+    private readonly captured;
+    private readonly leadIn?;
+    private input?;
+    private processing;
+    private sink?;
+    constructor(options?: WebAudioOptions);
     /** Make the player and unlock it. **Call it from the click**, before any `await`. */
     prime(): Promise<void>;
-    play(pcm: ArrayBuffer, frame: {
-        sample_rate: number;
-    }): void;
+    play(pcm: ArrayBuffer, frame: Pick<Frame, "sample_rate"> & Partial<Frame>): void;
     stop(): void;
     onDrained(fn: () => void): () => void;
+    now(): number;
+    heardTime(): number;
+    onScheduled(fn: (audio: ScheduledAudio) => void): () => void;
+    onStopped(fn: () => void): () => void;
+    onCapture(fn: (pcm: ArrayBuffer, rate: number) => void): () => void;
+    inputs(): Promise<AudioDevice[]>;
+    setInput(id?: string): void;
+    setProcessing(processing: Processing): void;
+    outputs(): Promise<AudioDevice[]>;
+    setOutputDevice(id: string): Promise<boolean>;
+    get canChooseOutput(): boolean;
     /** At the recorder's own rate, 24 kHz, whatever is asked. */
     startCapture(onPcm: (pcm: ArrayBuffer) => void, _options?: {
         rate?: number;
@@ -54,3 +98,4 @@ export declare class WebAudio implements AgentAudio {
     /** The conversation is over: the player's context goes too. */
     dispose(): void;
 }
+export {};

package/dist/web-audio/web-audio.js

@@ -5,20 +5,28 @@ const DUCKED = 0.3;
 export const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use push-to-talk.";
 /** Why it is refused in a browser that cannot cancel echo at all. */
 export const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use push-to-talk.";
+/** Why it is refused with echo cancellation turned off (`setProcessing`). */
+export const ECHO_OFF = "Echo cancellation is turned off, so the agent would hear itself. Turn it on, or use push-to-talk.";
 export class WebAudio {
-    constructor() {
+    constructor(options = {}) {
         this.player = null;
         this.recorder = null;
         this.onPcm = null;
         this.drained = new Set();
+        this.scheduled = new Set();
+        this.stopped = new Set();
+        this.captured = new Set();
+        if (options.leadInMs !== undefined)
+            this.leadIn = options.leadInMs / 1000;
+        this.input = options.input;
+        this.processing = options.processing ?? {};
     }
     /** Make the player and unlock it. **Call it from the click**, before any `await`. */
     prime() {
         this.player ?? (this.player = new PcmPlayer({
-            onIdle: () => {
-                for (const fn of this.drained)
-                    fn();
-            },
+            leadIn: this.leadIn,
+            sinkId: this.sink,
+            onIdle: () => tell(this.drained),
         }));
         return this.player.resume();
     }
@@ -27,21 +35,73 @@ export class WebAudio {
             void this.prime();
         const player = this.player;
         player.setSampleRate(frame.sample_rate);
-        player.push(pcm);
+        const queued = player.push(pcm);
+        if (queued) {
+            tell(this.scheduled, {
+                ...queued,
+                channels: frame.channels ?? 1,
+                chunk: frame.chunk ?? 0,
+                turn: frame.turn ?? "",
+            });
+        }
     }
     stop() {
-        this.player?.stop();
+        if (this.player?.stop())
+            tell(this.stopped);
     }
     onDrained(fn) {
-        this.drained.add(fn);
-        return () => {
-            this.drained.delete(fn);
-        };
+        return add(this.drained, fn);
+    }
+    now() {
+        return this.player?.now ?? 0;
+    }
+    heardTime() {
+        return Math.max(0, this.now() - (this.player?.latency ?? 0));
+    }
+    onScheduled(fn) {
+        return add(this.scheduled, fn);
+    }
+    onStopped(fn) {
+        return add(this.stopped, fn);
+    }
+    onCapture(fn) {
+        return add(this.captured, fn);
+    }
+    inputs() {
+        return devices("audioinput");
+    }
+    setInput(id) {
+        this.input = id;
+    }
+    setProcessing(processing) {
+        this.processing = processing;
+    }
+    async outputs() {
+        return this.canChooseOutput ? devices("audiooutput") : [];
+    }
+    async setOutputDevice(id) {
+        if (!this.canChooseOutput)
+            return false;
+        if (this.player && !(await this.player.setSink(id)))
+            return false;
+        this.sink = id;
+        return true;
+    }
+    get canChooseOutput() {
+        return (typeof AudioContext !== "undefined" &&
+            "setSinkId" in AudioContext.prototype);
     }
     /** At the recorder's own rate, 24 kHz, whatever is asked. */
     async startCapture(onPcm, _options) {
         const { PcmRecorder } = await import("./recorder.js");
-        const mic = new PcmRecorder({ onChunk: (pcm) => this.onPcm?.(pcm) });
+        const mic = new PcmRecorder({
+            input: this.input,
+            processing: this.processing,
+            onChunk: (pcm) => {
+                this.onPcm?.(pcm);
+                tell(this.captured, pcm, mic.rate);
+            },
+        });
         this.onPcm = onPcm;
         this.recorder = mic;
         try {
@@ -59,16 +119,24 @@ export class WebAudio {
         const deliver = this.onPcm;
         this.recorder = null;
         this.onPcm = null;
-        const tail = mic?.stop();
-        if (tail)
-            deliver?.(tail);
+        if (!mic)
+            return;
+        const rate = mic.rate;
+        const tail = mic.stop();
+        if (!tail)
+            return;
+        deliver?.(tail);
+        tell(this.captured, tail, rate);
     }
     duck(on) {
         this.player?.setVolume(on ? DUCKED : 1);
     }
     async handsFree() {
-        const devices = typeof navigator === "undefined" ? undefined : navigator.mediaDevices;
-        if (!devices?.getSupportedConstraints?.().echoCancellation) {
+        if (this.processing.echoCancellation === false) {
+            return { ok: false, reason: ECHO_OFF };
+        }
+        const media = mediaDevices();
+        if (!media?.getSupportedConstraints?.().echoCancellation) {
             return { ok: false, reason: NO_ECHO };
         }
         // iPadOS says it is a Mac: a Mac with a touch screen is an iPad.
@@ -83,3 +151,34 @@ export class WebAudio {
         this.player = null;
     }
 }
+const mediaDevices = () => typeof navigator === "undefined" ? undefined : navigator.mediaDevices;
+/** The browser's devices of `kind`; ids and labels empty until the microphone was allowed. */
+async function devices(kind) {
+    const listed = (await mediaDevices()?.enumerateDevices?.()) ?? [];
+    return listed
+        .filter((device) => device.kind === kind)
+        .map(({ deviceId, label }) => ({ id: deviceId, label }));
+}
+function add(listeners, fn) {
+    listeners.add(fn);
+    return () => {
+        listeners.delete(fn);
+    };
+}
+/**
+ * Call each of the app's handlers. One that throws is rethrown on its own
+ * turn of the event loop, where the app's error reporting sees it — not
+ * here, where it would stop the player or the recorder halfway.
+ */
+function tell(listeners, ...args) {
+    for (const fn of listeners) {
+        try {
+            fn(...args);
+        }
+        catch (error) {
+            setTimeout(() => {
+                throw error;
+            });
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lumavox/client",
-  "version": "0.14.1",
+  "version": "0.16.0",
   "description": "Talk to Lumavox agents from a browser, a React Native app or Node, with an end user's ephemeral key: conversations on the agent's socket, sessions, memory and relations. Zero dependencies.",
   "license": "MIT",
   "type": "module",