@lumavox/client 0.6.0 → 0.9.0

package/README.md

@@ -29,7 +29,9 @@ const outcome = await conversation.send("What should I see on my first morning?"
 // { status: "done", frame } | { status: "interrupted", frame } | { status: "dropped", reason, message }
 ```
 
-React and React Native: `@lumavox/react`, hooks over this.
+React: `@lumavox/react`, hooks over this. A React Native app:
+`@lumavox/react-native`, the hooks and this with push-to-talk audio, in one
+install.
 
 The guide, with every example: the docs' **SDKs** tab, *Client SDK*.
 
@@ -126,11 +128,27 @@ const turn = await conversation.endAudio();    // a spoken turn
 const words = await conversation.endAudio({ transcribeOnly: true });
 ```
 
-The SDK captures and plays nothing itself: a recorder hands it samples, a
-player takes `onAudio`'s. `floatToPcm16`, `pcm16ToFloat`, `resample`,
-`concatPcm` and `pcmSeconds` convert between them and the wire;
+`createClient` captures and plays nothing itself: a recorder hands it
+samples, a player takes `onAudio`'s. `floatToPcm16`, `pcm16ToFloat`,
+`resample`, `concatPcm` and `pcmSeconds` convert between them and the wire;
 `AgentAudio` is the interface `@lumavox/react` drives.
 
+### Audio in a browser
+
+```ts
+import { createWebAudio } from "@lumavox/client/web-audio";
+
+const audio = createWebAudio();
+startButton.onclick = () => audio.prime(); // in the click: browsers start sound only from one
+```
+
+An `AgentAudio` on Web Audio, its own import so the client carries none of
+it (~5.5 KB minified): replies play gaplessly as they arrive, at the
+device's own rate; the microphone records at 24 kHz, with the browser's
+echo cancellation, noise suppression and gain control. Hand it to
+`@lumavox/react`'s `useAgent`, or call `play`, `startCapture` and the rest
+yourself.
+
 ## Runtimes
 
 Browsers, React Native (Hermes, ES2020 — and each binary frame reaches JS

package/dist/audio.d.ts

@@ -1,12 +1,13 @@
 /**
  * What an app plugs in to make a conversation audible and to speak in it:
- * a player and a recorder. **Types only in v1**: the SDK captures and
- * plays nothing itself, so it adds no audio library to a customer's app —
- * the console's Web Audio player and recorder are the first implementation
- * (`apps/console/src/components/audio/agent-audio.ts`), and
- * `@lumavox/react`'s `useAgent` drives one: it plays every audio frame,
- * stops on a cancel, a mute or a new turn, and records at the rate the
- * recorder was granted. Voice activity detection stays the app's.
+ * a player and a recorder. `createClient` captures and plays nothing
+ * itself, so it adds no audio to a customer's app; a browser's is the
+ * optional `@lumavox/client/web-audio` (`src/web-audio/`), a phone's
+ * `@lumavox/react-native`'s, and an app may write its own. `@lumavox/react`'s
+ * `useAgent` drives one: it plays every audio frame, stops on a cancel, a
+ * mute or a new turn, records at the rate the recorder was granted, and
+ * throws away a recording the device interrupted.
+ * Voice activity detection stays the app's.
  */
 export interface AgentAudio {
     /** Queue one frame of a reply: 16-bit little-endian PCM, in wire order. */
@@ -35,4 +36,13 @@ export interface AgentAudio {
     stopCapture(): Promise<void>;
     /** Free everything: the conversation is over. */
     dispose(): void;
+    /**
+     * Optional: `fn(true)` when the device takes the audio away — a phone
+     * call, Siri, another app, the app leaving the screen — and `fn(false)`
+     * when it gives it back. The adapter suspends playback (a reply's frames
+     * keep queueing) and closes its recorder itself; whoever drives it throws
+     * away the recording that was open, its speaker cut off mid-sentence.
+     * Answers the unsubscribe.
+     */
+    onInterruption?(fn: (interrupted: boolean) => void): () => void;
 }

package/dist/events.d.ts

@@ -9,7 +9,7 @@
  * test (`apps/edge/test/sdk.test.ts`) holds every field of every frame the
  * real conversation object sends to these, both ways.
  */
-import type { ErrorCode, RelationInfo } from "./gen/index.js";
+import type { ErrorCode, MessageSource, RelationInfo } from "./gen/index.js";
 import type { ServerEventName } from "./protocol.gen.js";
 interface Frame<T extends ServerEventName> {
     type: T;
@@ -207,6 +207,8 @@ export interface TurnDoneFrame extends Frame<"turn.done">, InTurn {
     heard_seconds: number | null;
     /** Your tools it called, in order. */
     tool_calls: ToolCall[];
+    /** The web pages it read, with the agent's `url_context` on; `[]` for none. */
+    sources: MessageSource[];
     timings: TurnTimings;
 }
 /** The turn ended early; what was written before it stopped still stands. */

package/dist/gen/index.d.ts

@@ -1 +1 @@
-export type { ClientOptions, CompactionInfo, CompactRequest, CompactSessionData, CompactSessionError, CompactSessionErrors, CompactSessionResponse, CompactSessionResponses, CreateSessionData, CreateSessionError, CreateSessionErrors, CreateSessionResponse, CreateSessionResponses, DeleteAllFactsData, DeleteAllFactsError, DeleteAllFactsErrors, DeleteAllFactsResponse, DeleteAllFactsResponses, DeleteFactData, DeleteFactError, DeleteFactErrors, DeleteFactResponse, DeleteFactResponses, DeleteSessionData, DeleteSessionError, DeleteSessionErrors, DeleteSessionResponse, DeleteSessionResponses, ErrorCode, ErrorInfo, ErrorResponse, ExtractSessionFactsData, ExtractSessionFactsError, ExtractSessionFactsErrors, ExtractSessionFactsResponse, ExtractSessionFactsResponses, FactInfo, FactPage, FactsDeleted, GetSessionData, GetSessionError, GetSessionErrors, GetSessionResponse, GetSessionResponses, KnowledgeCitation, KnowledgeLookupInfo, ListFactsData, ListFactsError, ListFactsErrors, ListFactsResponse, ListFactsResponses, ListRelationsData, ListRelationsError, ListRelationsErrors, ListRelationsResponse, ListRelationsResponses, ListSessionsData, ListSessionsError, ListSessionsErrors, ListSessionsResponse, ListSessionsResponses, MessageInfo, MessageKnowledge, MessagePage, MessageToolCall, RelationInfo, RelationPage, SessionCreateRequest, SessionDetail, SessionInfo, SessionPage, ValidationIssue } from './types.gen.js';
+export type { ClientOptions, CompactionInfo, CompactRequest, CompactSessionData, CompactSessionError, CompactSessionErrors, CompactSessionResponse, CompactSessionResponses, CreateSessionData, CreateSessionError, CreateSessionErrors, CreateSessionResponse, CreateSessionResponses, DeleteAllFactsData, DeleteAllFactsError, DeleteAllFactsErrors, DeleteAllFactsResponse, DeleteAllFactsResponses, DeleteFactData, DeleteFactError, DeleteFactErrors, DeleteFactResponse, DeleteFactResponses, DeleteSessionData, DeleteSessionError, DeleteSessionErrors, DeleteSessionResponse, DeleteSessionResponses, ErrorCode, ErrorInfo, ErrorResponse, ExtractSessionFactsData, ExtractSessionFactsError, ExtractSessionFactsErrors, ExtractSessionFactsResponse, ExtractSessionFactsResponses, FactInfo, FactPage, FactsDeleted, GetSessionData, GetSessionError, GetSessionErrors, GetSessionResponse, GetSessionResponses, KnowledgeCitation, KnowledgeLookupInfo, ListFactsData, ListFactsError, ListFactsErrors, ListFactsResponse, ListFactsResponses, ListRelationsData, ListRelationsError, ListRelationsErrors, ListRelationsResponse, ListRelationsResponses, ListSessionsData, ListSessionsError, ListSessionsErrors, ListSessionsResponse, ListSessionsResponses, MessageInfo, MessageKnowledge, MessagePage, MessageSource, MessageToolCall, RelationInfo, RelationPage, SessionCreateRequest, SessionDetail, SessionInfo, SessionPage, ValidationIssue } from './types.gen.js';

package/dist/gen/types.gen.d.ts

@@ -89,6 +89,10 @@ export type MessageInfo = {
      */
     mood_intensity: number | null;
     role: 'user' | 'assistant';
+    /**
+     * On a reply: the web pages it read, with the agent's `url_context` switch on. Empty when it read none, and on the person's messages.
+     */
+    sources: Array<MessageSource>;
     /**
      * On a reply: the calls of your tools it made, in order, each with its arguments and what it answered. Empty when it called none, and on the person's messages.
      */
@@ -118,6 +122,16 @@ export type KnowledgeCitation = {
     score: number;
     title: string;
 };
+/**
+ * A web page the reply read with the agent's `url_context` switch on.
+ */
+export type MessageSource = {
+    url: string;
+    /**
+     * Whether the page could be read. A reply may name a page it could not fetch — behind a login, refused by its site, gone — and answers without it.
+     */
+    retrieved: boolean;
+};
 export type MessageToolCall = {
     /**
      * The call's id, `call_…`: what your server or app was sent.

package/dist/web-audio/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * `@lumavox/client/web-audio`: a browser's player and recorder for a
+ * conversation, on Web Audio — an `AgentAudio` that `@lumavox/react`'s
+ * `useAgent` drives. An entry of its own, so `createClient` carries none of
+ * it (`test/dist.test.ts` holds both sizes); zero dependencies, as the rest.
+ *
+ * ```ts
+ * const audio = createWebAudio();
+ * button.onclick = () => audio.prime(); // in the click: browsers start audio only from one
+ * ```
+ */
+import { WebAudio } from "./web-audio.js";
+export type { WebAudio };
+/** A player and a recorder for one conversation. Call `prime()` from the click that starts it. */
+export declare function createWebAudio(): WebAudio;

package/dist/web-audio/index.js

@@ -0,0 +1,16 @@
+/**
+ * `@lumavox/client/web-audio`: a browser's player and recorder for a
+ * conversation, on Web Audio — an `AgentAudio` that `@lumavox/react`'s
+ * `useAgent` drives. An entry of its own, so `createClient` carries none of
+ * it (`test/dist.test.ts` holds both sizes); zero dependencies, as the rest.
+ *
+ * ```ts
+ * const audio = createWebAudio();
+ * button.onclick = () => audio.prime(); // in the click: browsers start audio only from one
+ * ```
+ */
+import { WebAudio } from "./web-audio.js";
+/** A player and a recorder for one conversation. Call `prime()` from the click that starts it. */
+export function createWebAudio() {
+    return new WebAudio();
+}

package/dist/web-audio/player.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Playing raw PCM as it arrives, with no container and no `<audio>` element.
+ *
+ * **Why not `<audio>`.** That needs a container — a data URL with a MIME
+ * type — and the socket sends bare samples precisely to avoid one: base64 is
+ * 33% of every audio byte, and audio is most of a spoken turn. There is also
+ * nothing to point an `<audio>` element *at* until a chunk is complete, which
+ * would give back the latency the transport exists to win.
+ *
+ * So: Web Audio, one `AudioBufferSourceNode` per blob, each scheduled to start
+ * exactly where the last one ended. Three things make that sound continuous
+ * rather than like a stutter of clips:
+ *
+ *  - **The schedule is kept in the context's own clock**, never in wall time.
+ *    `playhead` is an absolute `AudioContext.currentTime`, so scheduling is
+ *    sample-accurate and drift-free even if the tab is busy.
+ *  - **A blob arriving late restarts from `currentTime`**, not from a playhead
+ *    in the past — otherwise it is scheduled to have already happened and is
+ *    simply dropped, which is heard as a missing sentence.
+ *  - **A tiny lead-in** (`SCHEDULE_AHEAD`) absorbs the jitter between a frame
+ *    arriving and it being decoded, at the cost of a few milliseconds.
+ *
+ * **The context runs at the device's own rate**; each buffer is made at the
+ * stream's (24 kHz unless `audio.start` says otherwise) and the browser
+ * resamples it as it plays — every browser does, as Web Audio requires.
+ * Firefox's echo canceller hears only a context at the device's rate, which
+ * hands-free will need; until the console's code became this module (client
+ * 0.8.0) the context was pinned to the stream's rate and rebuilt when it
+ * changed.
+ *
+ * Nothing here decodes: the format is 16-bit signed little-endian, so it is
+ * an `Int16Array` read straight into a `Float32Array`.
+ */
+export interface PcmPlayerOptions {
+    /** Hz of the PCM pushed, until `setSampleRate`: the socket's default. */
+    sampleRate?: number;
+    channels?: number;
+    /** Called when the queue drains — every scheduled source has finished. */
+    onIdle?: () => void;
+}
+export declare class PcmPlayer {
+    private context;
+    private gain;
+    private playhead;
+    /** A trailing byte from the previous frame, when one was split. */
+    private remainder;
+    private scheduled;
+    /** Bumped by `stop()`, so a source cut off by it does not count itself out of
+     *  a queue that has already been zeroed — `scheduled` went negative after a
+     *  barge-in until console 0.197.0, and `playing` lied from then on. */
+    private generation;
+    private sampleRate;
+    private readonly channels;
+    private readonly onIdle?;
+    constructor(options?: PcmPlayerOptions);
+    /**
+     * The rate of the PCM pushed from here on: each `audio.start` carries its
+     * window's own, since an agent's voice may be edited mid-conversation.
+     * What is queued plays on at its own rate — every buffer carries it — so
+     * nothing is dropped and the context stays.
+     */
+    setSampleRate(rate: number): void;
+    /**
+     * Create the context, or resume one the browser suspended.
+     *
+     * **Must be called from a user gesture the first time.** Every browser
+     * suspends a context created outside one, and a suspended context accepts
+     * every scheduling call silently — so the symptom is a conversation that
+     * looks like it is speaking and makes no sound.
+     */
+    resume(): Promise<void>;
+    /** Whether anything is scheduled and still to play. */
+    get playing(): boolean;
+    /**
+     * Queue one blob of PCM.
+     *
+     * Ignored before {@link resume} — a caller that pushes audio into a player
+     * nobody unlocked has a gesture problem, and buffering it would hide that
+     * until the sound arrived minutes later in one burst.
+     */
+    push(pcm: ArrayBuffer): void;
+    /**
+     * Stop everything now — barge-in, or the turn being cancelled.
+     *
+     * Disconnecting the gain node rather than stopping each source: sources are
+     * fire-and-forget once scheduled and holding references to them purely to
+     * stop them is a leak waiting to happen. A fresh node is built on the next
+     * push.
+     */
+    stop(): void;
+    /** Release the audio device. Safe to call more than once. */
+    close(): Promise<void>;
+}

package/dist/web-audio/player.js

@@ -0,0 +1,187 @@
+/**
+ * Playing raw PCM as it arrives, with no container and no `<audio>` element.
+ *
+ * **Why not `<audio>`.** That needs a container — a data URL with a MIME
+ * type — and the socket sends bare samples precisely to avoid one: base64 is
+ * 33% of every audio byte, and audio is most of a spoken turn. There is also
+ * nothing to point an `<audio>` element *at* until a chunk is complete, which
+ * would give back the latency the transport exists to win.
+ *
+ * So: Web Audio, one `AudioBufferSourceNode` per blob, each scheduled to start
+ * exactly where the last one ended. Three things make that sound continuous
+ * rather than like a stutter of clips:
+ *
+ *  - **The schedule is kept in the context's own clock**, never in wall time.
+ *    `playhead` is an absolute `AudioContext.currentTime`, so scheduling is
+ *    sample-accurate and drift-free even if the tab is busy.
+ *  - **A blob arriving late restarts from `currentTime`**, not from a playhead
+ *    in the past — otherwise it is scheduled to have already happened and is
+ *    simply dropped, which is heard as a missing sentence.
+ *  - **A tiny lead-in** (`SCHEDULE_AHEAD`) absorbs the jitter between a frame
+ *    arriving and it being decoded, at the cost of a few milliseconds.
+ *
+ * **The context runs at the device's own rate**; each buffer is made at the
+ * stream's (24 kHz unless `audio.start` says otherwise) and the browser
+ * resamples it as it plays — every browser does, as Web Audio requires.
+ * Firefox's echo canceller hears only a context at the device's rate, which
+ * hands-free will need; until the console's code became this module (client
+ * 0.8.0) the context was pinned to the stream's rate and rebuilt when it
+ * changed.
+ *
+ * Nothing here decodes: the format is 16-bit signed little-endian, so it is
+ * an `Int16Array` read straight into a `Float32Array`.
+ */
+import { DEFAULT_SAMPLE_RATE } from "../protocol.gen.js";
+/** How far ahead of `currentTime` a freshly-started run begins. Enough to
+ *  absorb frame jitter, short enough not to be perceived as delay. */
+const SCHEDULE_AHEAD = 0.06;
+/** Divisor from 16-bit signed to the -1..1 floats Web Audio wants. 32768,
+ *  because the negative range is one larger than the positive one. */
+const FULL_SCALE = 32768;
+export class PcmPlayer {
+    constructor(options = {}) {
+        this.context = null;
+        this.gain = null;
+        this.playhead = 0;
+        /** A trailing byte from the previous frame, when one was split. */
+        this.remainder = new Uint8Array(0);
+        this.scheduled = 0;
+        /** Bumped by `stop()`, so a source cut off by it does not count itself out of
+         *  a queue that has already been zeroed — `scheduled` went negative after a
+         *  barge-in until console 0.197.0, and `playing` lied from then on. */
+        this.generation = 0;
+        // The edge's default (the vocabulary's), until an `audio.start` says
+        // otherwise.
+        this.sampleRate = options.sampleRate ?? DEFAULT_SAMPLE_RATE;
+        this.channels = options.channels ?? 1;
+        this.onIdle = options.onIdle;
+    }
+    /**
+     * The rate of the PCM pushed from here on: each `audio.start` carries its
+     * window's own, since an agent's voice may be edited mid-conversation.
+     * What is queued plays on at its own rate — every buffer carries it — so
+     * nothing is dropped and the context stays.
+     */
+    setSampleRate(rate) {
+        if (!Number.isFinite(rate) || rate <= 0)
+            return;
+        this.sampleRate = rate;
+    }
+    /**
+     * Create the context, or resume one the browser suspended.
+     *
+     * **Must be called from a user gesture the first time.** Every browser
+     * suspends a context created outside one, and a suspended context accepts
+     * every scheduling call silently — so the symptom is a conversation that
+     * looks like it is speaking and makes no sound.
+     */
+    async resume() {
+        if (!this.context) {
+            this.context = new AudioContext();
+            this.gain = this.context.createGain();
+            this.gain.connect(this.context.destination);
+        }
+        if (this.context.state === "suspended")
+            await this.context.resume();
+    }
+    /** Whether anything is scheduled and still to play. */
+    get playing() {
+        return this.scheduled > 0;
+    }
+    /**
+     * Queue one blob of PCM.
+     *
+     * Ignored before {@link resume} — a caller that pushes audio into a player
+     * nobody unlocked has a gesture problem, and buffering it would hide that
+     * until the sound arrived minutes later in one burst.
+     */
+    push(pcm) {
+        const context = this.context;
+        if (!context)
+            return;
+        // **A split sample is carried, never dropped.** The edge sends whole
+        // samples per frame (`apps/edge/src/voice/tts.ts` holds an odd byte for
+        // the next blob), so this should never fire — but truncating instead is
+        // the bug that made ElevenLabs playback white noise, and it is worth being
+        // unable to reintroduce. Dropping the odd byte does not lose one sample,
+        // it shifts every byte after it by one and turns the rest of the frame
+        // into noise until something realigns it.
+        let bytes = new Uint8Array(pcm);
+        if (this.remainder.length) {
+            const joined = new Uint8Array(this.remainder.length + bytes.length);
+            joined.set(this.remainder, 0);
+            joined.set(bytes, this.remainder.length);
+            bytes = joined;
+        }
+        const usable = bytes.length - (bytes.length % 2);
+        this.remainder = bytes.slice(usable);
+        if (usable < 2)
+            return;
+        // Copied into its own buffer rather than viewed in place: `Int16Array`
+        // needs a 2-byte-aligned offset, and a frame joined to a carried byte has
+        // no such promise.
+        const samples = new Int16Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + usable));
+        // At the stream's rate, not the context's: the browser resamples it.
+        const buffer = context.createBuffer(this.channels, samples.length / this.channels, this.sampleRate);
+        for (let channel = 0; channel < this.channels; channel += 1) {
+            const target = buffer.getChannelData(channel);
+            for (let i = 0; i < target.length; i += 1)
+                target[i] = samples[i * this.channels + channel] / FULL_SCALE;
+        }
+        const source = context.createBufferSource();
+        source.buffer = buffer;
+        source.connect(this.gain);
+        // **The branch that keeps it gapless.** While audio is still queued, the
+        // next blob starts exactly where the last one ends; once the queue has run
+        // dry, the playhead is in the past and scheduling against it would mean
+        // "start before now", which the API resolves as "start immediately" for the
+        // first blob and drops for every one after it.
+        const now = context.currentTime;
+        const startAt = this.playhead > now ? this.playhead : now + SCHEDULE_AHEAD;
+        source.start(startAt);
+        this.playhead = startAt + buffer.duration;
+        this.scheduled += 1;
+        const generation = this.generation;
+        source.onended = () => {
+            if (generation !== this.generation)
+                return;
+            this.scheduled -= 1;
+            if (this.scheduled === 0)
+                this.onIdle?.();
+        };
+    }
+    /**
+     * Stop everything now — barge-in, or the turn being cancelled.
+     *
+     * Disconnecting the gain node rather than stopping each source: sources are
+     * fire-and-forget once scheduled and holding references to them purely to
+     * stop them is a leak waiting to happen. A fresh node is built on the next
+     * push.
+     */
+    stop() {
+        if (!this.context || !this.gain)
+            return;
+        this.gain.disconnect();
+        this.gain = this.context.createGain();
+        this.gain.connect(this.context.destination);
+        this.playhead = 0;
+        const wasPlaying = this.scheduled > 0;
+        this.scheduled = 0;
+        this.generation += 1;
+        // A half sample belonging to audio that will never play now.
+        this.remainder = new Uint8Array(0);
+        if (wasPlaying)
+            this.onIdle?.();
+    }
+    /** Release the audio device. Safe to call more than once. */
+    async close() {
+        const context = this.context;
+        this.context = null;
+        this.gain = null;
+        this.playhead = 0;
+        this.scheduled = 0;
+        this.remainder = new Uint8Array(0);
+        if (context && context.state !== "closed")
+            await context.close();
+    }
+}

package/dist/web-audio/recorder.d.ts

@@ -0,0 +1,114 @@
+/**
+ * The microphone, as the same raw PCM this socket already plays.
+ *
+ * The mirror of `player.ts`: 16-bit signed little-endian mono, at 24 kHz —
+ * the rate the socket's audio defaults to (`DEFAULT_SAMPLE_RATE`), and one
+ * of `RECORDING_RATES`.
+ *
+ * **No `MediaRecorder`, and that is the whole design.** It produces a
+ * *container* — webm/opus, whatever the browser feels like — which the socket
+ * cannot take: binary frames there are bare samples with no header, and the
+ * server would have to decode a container to find out what it was given. Web
+ * Audio hands over float samples instead, and converting those to 16-bit
+ * integers is four lines.
+ *
+ * Three things make it work in practice:
+ *
+ *  - **This context is opened at 24 kHz** rather than resampled after the
+ *    fact. Browsers accept a `sampleRate` on the constructor and do the
+ *    conversion in the audio thread, which is both better and free. (The
+ *    player's context is the device's own: `player.ts` says why.)
+ *  - **`AudioWorklet` with a `ScriptProcessor` fallback.** The worklet runs off
+ *    the main thread, so a busy React render cannot drop samples — which is
+ *    heard as a clipped word rather than as a stutter, and is exactly the kind
+ *    of fault that would be blamed on the recogniser.
+ *  - **The track is stopped on `stop()`**, not merely disconnected. A live
+ *    `MediaStreamTrack` leaves the browser's recording indicator on, and a
+ *    microphone that looks like it is still listening is a trust problem
+ *    whatever the code is doing.
+ *
+ * Loaded on the first press (`web-audio.ts`), so a page that never records
+ * never parses it.
+ */
+/** The rate this records at, whatever is asked: the socket's default. */
+export declare const RECORD_SAMPLE_RATE = 24000;
+/** Samples per delivered chunk — about 85 ms at 24 kHz. Small enough that the
+ *  upload overlaps the talking, large enough not to be one WebSocket frame per
+ *  animation tick.
+ *
+ *  **Applied to the worklet path too.** The worklet posts every 128-sample
+ *  render quantum; one WebSocket frame a post is about 188 frames a second of
+ *  256 bytes, each a TLS record and a message the end user DO wakes for —
+ *  sixteen times as many as at this size. Batched here, on the main thread,
+ *  where the tail can be flushed synchronously when the button is released. */
+export declare const FRAME_SAMPLES = 2048;
+export interface PcmRecorderOptions {
+    /** Called with each block of samples, already 16-bit little-endian. */
+    onChunk: (pcm: ArrayBuffer) => void;
+    sampleRate?: number;
+}
+/** Float samples to 16-bit signed little-endian, which is what the wire wants. */
+export declare function toPcm16(samples: Float32Array): ArrayBuffer;
+export declare class PcmRecorder {
+    /** Whether chunks may still be delivered.
+     *
+     *  **Cleared synchronously by `stop()`, and checked before every delivery.**
+     *  Disconnecting a node does not un-post the messages it has already sent:
+     *  the worklet runs on the audio thread and its `port.postMessage` calls
+     *  arrive on the main thread as ordinary tasks, so a handful land *after*
+     *  `stop()` has returned.
+     *
+     *  That is not a tidiness problem. The caller sends each chunk into an open
+     *  `user_audio` window and closes that window immediately after stopping —
+     *  so a late chunk is binary audio arriving with no window open, which the
+     *  edge answers by closing the connection. That is how it shows: hold,
+     *  release, hold, release, and the socket dies with
+     *  "Binary audio arrived with no user_audio.start before it."
+     */
+    private live;
+    private context;
+    private stream;
+    private node;
+    private source;
+    private readonly onChunk;
+    private readonly sampleRate;
+    /** Samples not yet delivered — see `FRAME_SAMPLES`. */
+    private pending;
+    private pendingSamples;
+    constructor(options: PcmRecorderOptions);
+    /** True once `start()` has a live microphone and has not been stopped. */
+    get recording(): boolean;
+    /**
+     * Hz the delivered samples actually are.
+     *
+     * The context is *asked* for `sampleRate`; browsers honour it or throw, and
+     * this reads back what was granted rather than repeating the request. It is
+     * what a caller must declare on `user_audio.start` — a declared rate the
+     * samples are not is the wrong pitch and a wrong bill, behind a 200.
+     */
+    get rate(): number;
+    /**
+     * Ask for the microphone and begin delivering samples.
+     *
+     * Rejects if permission is refused, which is the caller's to report — a mic
+     * button that silently does nothing after a denied prompt is the worst of the
+     * available outcomes.
+     */
+    start(): Promise<void>;
+    /**
+     * Stop, release the microphone, and turn the browser's indicator off.
+     *
+     * Returns the **tail**: samples batched and not yet delivered when the button
+     * was released — up to one frame, ~85 ms, the end of the last word. Returned
+     * rather than delivered through `onChunk`, because the caller has by now
+     * stopped accepting chunks (`stop()` is what it calls to stop them) and
+     * decides for itself whether the tail goes on the wire before `user_audio.end`.
+     * `null` when there is nothing held.
+     */
+    stop(): ArrayBuffer | null;
+    /** Batch a block of samples, delivering whole frames as they fill. */
+    private deliver;
+    /** Up to `count` pending samples (all of them by default), as one block. */
+    private takePending;
+    private tap;
+}

package/dist/web-audio/recorder.js

@@ -0,0 +1,255 @@
+/**
+ * The microphone, as the same raw PCM this socket already plays.
+ *
+ * The mirror of `player.ts`: 16-bit signed little-endian mono, at 24 kHz —
+ * the rate the socket's audio defaults to (`DEFAULT_SAMPLE_RATE`), and one
+ * of `RECORDING_RATES`.
+ *
+ * **No `MediaRecorder`, and that is the whole design.** It produces a
+ * *container* — webm/opus, whatever the browser feels like — which the socket
+ * cannot take: binary frames there are bare samples with no header, and the
+ * server would have to decode a container to find out what it was given. Web
+ * Audio hands over float samples instead, and converting those to 16-bit
+ * integers is four lines.
+ *
+ * Three things make it work in practice:
+ *
+ *  - **This context is opened at 24 kHz** rather than resampled after the
+ *    fact. Browsers accept a `sampleRate` on the constructor and do the
+ *    conversion in the audio thread, which is both better and free. (The
+ *    player's context is the device's own: `player.ts` says why.)
+ *  - **`AudioWorklet` with a `ScriptProcessor` fallback.** The worklet runs off
+ *    the main thread, so a busy React render cannot drop samples — which is
+ *    heard as a clipped word rather than as a stutter, and is exactly the kind
+ *    of fault that would be blamed on the recogniser.
+ *  - **The track is stopped on `stop()`**, not merely disconnected. A live
+ *    `MediaStreamTrack` leaves the browser's recording indicator on, and a
+ *    microphone that looks like it is still listening is a trust problem
+ *    whatever the code is doing.
+ *
+ * Loaded on the first press (`web-audio.ts`), so a page that never records
+ * never parses it.
+ */
+/** The rate this records at, whatever is asked: the socket's default. */
+export const RECORD_SAMPLE_RATE = 24000;
+/** Multiplier from the -1..1 floats Web Audio gives to 16-bit signed. 32767,
+ *  not 32768, so a full-scale sample cannot wrap to the opposite sign. */
+const FULL_SCALE = 32767;
+/** Samples per delivered chunk — about 85 ms at 24 kHz. Small enough that the
+ *  upload overlaps the talking, large enough not to be one WebSocket frame per
+ *  animation tick.
+ *
+ *  **Applied to the worklet path too.** The worklet posts every 128-sample
+ *  render quantum; one WebSocket frame a post is about 188 frames a second of
+ *  256 bytes, each a TLS record and a message the end user DO wakes for —
+ *  sixteen times as many as at this size. Batched here, on the main thread,
+ *  where the tail can be flushed synchronously when the button is released. */
+export const FRAME_SAMPLES = 2048;
+const WORKLET = `
+class Tap extends AudioWorkletProcessor {
+  process(inputs) {
+    const channel = inputs[0]?.[0];
+    // A disconnected or silent-by-design input yields an empty array rather
+    // than zeroes; forwarding that would post empty frames forever.
+    if (channel && channel.length) this.port.postMessage(channel.slice());
+    return true;
+  }
+}
+registerProcessor("pcm-tap", Tap);
+`;
+/** Float samples to 16-bit signed little-endian, which is what the wire wants. */
+export function toPcm16(samples) {
+    const out = new DataView(new ArrayBuffer(samples.length * 2));
+    for (let i = 0; i < samples.length; i++) {
+        // Clamped: a sample outside -1..1 is legal in Web Audio and wraps rather
+        // than clips once it is an integer, which is heard as a loud crack.
+        const clamped = Math.max(-1, Math.min(1, samples[i]));
+        out.setInt16(i * 2, Math.round(clamped * FULL_SCALE), true);
+    }
+    return out.buffer;
+}
+export class PcmRecorder {
+    constructor(options) {
+        /** Whether chunks may still be delivered.
+         *
+         *  **Cleared synchronously by `stop()`, and checked before every delivery.**
+         *  Disconnecting a node does not un-post the messages it has already sent:
+         *  the worklet runs on the audio thread and its `port.postMessage` calls
+         *  arrive on the main thread as ordinary tasks, so a handful land *after*
+         *  `stop()` has returned.
+         *
+         *  That is not a tidiness problem. The caller sends each chunk into an open
+         *  `user_audio` window and closes that window immediately after stopping —
+         *  so a late chunk is binary audio arriving with no window open, which the
+         *  edge answers by closing the connection. That is how it shows: hold,
+         *  release, hold, release, and the socket dies with
+         *  "Binary audio arrived with no user_audio.start before it."
+         */
+        this.live = false;
+        this.context = null;
+        this.stream = null;
+        this.node = null;
+        this.source = null;
+        /** Samples not yet delivered — see `FRAME_SAMPLES`. */
+        this.pending = [];
+        this.pendingSamples = 0;
+        this.onChunk = options.onChunk;
+        this.sampleRate = options.sampleRate ?? RECORD_SAMPLE_RATE;
+    }
+    /** True once `start()` has a live microphone and has not been stopped. */
+    get recording() {
+        return this.live;
+    }
+    /**
+     * Hz the delivered samples actually are.
+     *
+     * The context is *asked* for `sampleRate`; browsers honour it or throw, and
+     * this reads back what was granted rather than repeating the request. It is
+     * what a caller must declare on `user_audio.start` — a declared rate the
+     * samples are not is the wrong pitch and a wrong bill, behind a 200.
+     */
+    get rate() {
+        return this.context?.sampleRate ?? this.sampleRate;
+    }
+    /**
+     * Ask for the microphone and begin delivering samples.
+     *
+     * Rejects if permission is refused, which is the caller's to report — a mic
+     * button that silently does nothing after a denied prompt is the worst of the
+     * available outcomes.
+     */
+    async start() {
+        if (this.stream)
+            return;
+        this.stream = await navigator.mediaDevices.getUserMedia({
+            audio: {
+                // The browser's own cleanup, which is better than anything worth doing
+                // here and costs nothing: a recogniser fed room echo transcribes it.
+                echoCancellation: true,
+                noiseSuppression: true,
+                autoGainControl: true,
+                channelCount: 1,
+            },
+        });
+        try {
+            this.context = new AudioContext({ sampleRate: this.sampleRate });
+            // Autoplay policy suspends a context created outside a gesture. Pressing
+            // a microphone button *is* one, so this resolves immediately — it is here
+            // for the case where the button was held from a restored tab.
+            if (this.context.state === "suspended")
+                await this.context.resume();
+            this.source = this.context.createMediaStreamSource(this.stream);
+            this.node = await this.tap(this.context);
+        }
+        catch (failure) {
+            // **The track is released on the way out** (console 0.197.0).
+            // `getUserMedia` had succeeded, so a context that refused the rate or a
+            // worklet that failed to load used to leave a live microphone —
+            // indicator on — behind an error the caller reported as "could not
+            // open the microphone".
+            this.stop();
+            throw failure;
+        }
+        // **Last, after everything that can fail.** Connecting is what starts the
+        // samples flowing, so anything between it and here would be a window in
+        // which chunks arrive before the caller has been told recording began.
+        this.live = true;
+        this.source.connect(this.node);
+        // **Not connected to the destination.** Routing the microphone to the
+        // speakers is feedback, and a `ScriptProcessorNode` needs a downstream to
+        // be pulled at all — so that fallback connects to a muted gain instead.
+    }
+    /**
+     * Stop, release the microphone, and turn the browser's indicator off.
+     *
+     * Returns the **tail**: samples batched and not yet delivered when the button
+     * was released — up to one frame, ~85 ms, the end of the last word. Returned
+     * rather than delivered through `onChunk`, because the caller has by now
+     * stopped accepting chunks (`stop()` is what it calls to stop them) and
+     * decides for itself whether the tail goes on the wire before `user_audio.end`.
+     * `null` when there is nothing held.
+     */
+    stop() {
+        // **First, before anything else.** Everything below is asynchronous in
+        // effect — a disconnected node's already-posted messages still arrive — and
+        // this is the only part that takes effect immediately.
+        this.live = false;
+        const tail = this.pendingSamples ? toPcm16(this.takePending()) : null;
+        this.source?.disconnect();
+        this.node?.disconnect();
+        for (const track of this.stream?.getTracks() ?? [])
+            track.stop();
+        void this.context?.close();
+        this.source = null;
+        this.node = null;
+        this.stream = null;
+        this.context = null;
+        return tail;
+    }
+    /** Batch a block of samples, delivering whole frames as they fill. */
+    deliver(samples) {
+        if (!this.live)
+            return;
+        this.pending.push(samples);
+        this.pendingSamples += samples.length;
+        while (this.pendingSamples >= FRAME_SAMPLES) {
+            this.onChunk(toPcm16(this.takePending(FRAME_SAMPLES)));
+        }
+    }
+    /** Up to `count` pending samples (all of them by default), as one block. */
+    takePending(count = this.pendingSamples) {
+        const out = new Float32Array(Math.min(count, this.pendingSamples));
+        let filled = 0;
+        while (filled < out.length) {
+            const head = this.pending[0];
+            const take = Math.min(head.length, out.length - filled);
+            out.set(head.subarray(0, take), filled);
+            filled += take;
+            if (take === head.length)
+                this.pending.shift();
+            else
+                this.pending[0] = head.subarray(take);
+        }
+        this.pendingSamples -= out.length;
+        return out;
+    }
+    async tap(context) {
+        if (context.audioWorklet) {
+            try {
+                const url = URL.createObjectURL(new Blob([WORKLET], { type: "application/javascript" }));
+                try {
+                    await context.audioWorklet.addModule(url);
+                }
+                finally {
+                    // Revoked as soon as it is loaded: the module is compiled by then,
+                    // and a blob URL left behind is a leak for the tab's lifetime.
+                    URL.revokeObjectURL(url);
+                }
+                const node = new AudioWorkletNode(context, "pcm-tap");
+                node.port.onmessage = (event) => {
+                    this.deliver(event.data);
+                };
+                return node;
+            }
+            catch {
+                // swallowed: fall through. Worklets are unavailable over plain HTTP on
+                // some browsers and the fallback is correct, just less pleasant.
+            }
+        }
+        // `ScriptProcessorNode` is deprecated and still universally supported. It
+        // runs on the main thread, so a heavy render can drop a block — acceptable
+        // as a fallback, not as the default.
+        const node = context.createScriptProcessor(FRAME_SAMPLES, 1, 0);
+        node.onaudioprocess = (event) => {
+            // Copied: the buffer is the node's and is reused for the next block.
+            this.deliver(event.inputBuffer.getChannelData(0).slice());
+        };
+        // Silent sink: without a downstream connection the node is never pulled and
+        // `onaudioprocess` never fires, which reads as a dead microphone.
+        const sink = context.createGain();
+        sink.gain.value = 0;
+        node.connect(sink);
+        sink.connect(context.destination);
+        return node;
+    }
+}

package/dist/web-audio/web-audio.d.ts

@@ -0,0 +1,40 @@
+/**
+ * The browser's player and recorder as an `AgentAudio`: what
+ * `@lumavox/react`'s `useAgent` plays every reply through and records with
+ * — the console's since client 0.8.0, which it was written for.
+ *
+ * - **The player is made on the first click** (`prime`): every browser
+ *   suspends an `AudioContext` created outside a user gesture, and a
+ *   suspended one accepts every scheduling call silently.
+ * - **Each frame says its rate**: `audio.start` carries the window's own
+ *   (the agent's voice is re-read every turn), and the player makes each
+ *   buffer at it.
+ * - **The recorder is loaded on the first press**: `recorder.ts` holds the
+ *   AudioWorklet setup, kept out of a page that never records. It records
+ *   at 24 kHz whatever is asked — a rate the socket takes. Its tail — the
+ *   last partial frame, the end of the last word — is delivered before
+ *   `stopCapture` settles, or the transcript never gets it.
+ */
+import type { AgentAudio } from "../audio.js";
+export declare class WebAudio implements AgentAudio {
+    private player;
+    private recorder;
+    private onPcm;
+    private readonly drained;
+    /** Make the player and unlock it. **Call it from the click**, before any `await`. */
+    prime(): Promise<void>;
+    play(pcm: ArrayBuffer, frame: {
+        sample_rate: number;
+    }): void;
+    stop(): void;
+    onDrained(fn: () => void): () => void;
+    /** At the recorder's own rate, 24 kHz, whatever is asked. */
+    startCapture(onPcm: (pcm: ArrayBuffer) => void, _options?: {
+        rate?: number;
+    }): Promise<{
+        rate: number;
+    }>;
+    stopCapture(): Promise<void>;
+    /** The conversation is over: the player's context goes too. */
+    dispose(): void;
+}

package/dist/web-audio/web-audio.js

@@ -0,0 +1,66 @@
+import { PcmPlayer } from "./player.js";
+export class WebAudio {
+    constructor() {
+        this.player = null;
+        this.recorder = null;
+        this.onPcm = null;
+        this.drained = new Set();
+    }
+    /** Make the player and unlock it. **Call it from the click**, before any `await`. */
+    prime() {
+        this.player ?? (this.player = new PcmPlayer({
+            onIdle: () => {
+                for (const fn of this.drained)
+                    fn();
+            },
+        }));
+        return this.player.resume();
+    }
+    play(pcm, frame) {
+        if (!this.player)
+            void this.prime();
+        const player = this.player;
+        player.setSampleRate(frame.sample_rate);
+        player.push(pcm);
+    }
+    stop() {
+        this.player?.stop();
+    }
+    onDrained(fn) {
+        this.drained.add(fn);
+        return () => {
+            this.drained.delete(fn);
+        };
+    }
+    /** At the recorder's own rate, 24 kHz, whatever is asked. */
+    async startCapture(onPcm, _options) {
+        const { PcmRecorder } = await import("./recorder.js");
+        const mic = new PcmRecorder({ onChunk: (pcm) => this.onPcm?.(pcm) });
+        this.onPcm = onPcm;
+        this.recorder = mic;
+        try {
+            await mic.start();
+        }
+        catch (failure) {
+            if (this.recorder === mic)
+                this.recorder = null;
+            throw failure;
+        }
+        return { rate: mic.rate };
+    }
+    async stopCapture() {
+        const mic = this.recorder;
+        const deliver = this.onPcm;
+        this.recorder = null;
+        this.onPcm = null;
+        const tail = mic?.stop();
+        if (tail)
+            deliver?.(tail);
+    }
+    /** The conversation is over: the player's context goes too. */
+    dispose() {
+        void this.stopCapture();
+        void this.player?.close();
+        this.player = null;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lumavox/client",
-  "version": "0.6.0",
+  "version": "0.9.0",
   "description": "Talk to Lumavox agents from a browser, a React Native app or Node, with an end user's ephemeral key: conversations on the agent's socket, sessions, memory and relations. Zero dependencies.",
   "license": "MIT",
   "type": "module",
@@ -16,6 +16,11 @@
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
     },
+    "./web-audio": {
+      "@ino/source": "./src/web-audio/index.ts",
+      "types": "./dist/web-audio/index.d.ts",
+      "default": "./dist/web-audio/index.js"
+    },
     "./package.json": "./package.json"
   },
   "scripts": {
@@ -27,14 +32,18 @@
     "format": "biome format --write",
     "lint": "biome lint",
     "test": "vitest run",
-    "test:unit": "vitest run",
+    "test:unit": "vitest run --project unit",
+    "test:browser": "vitest run --project browser",
     "typecheck": "tsc --noEmit && tsc --noEmit -p test"
   },
   "devDependencies": {
     "@biomejs/biome": "2.6.2",
     "@hey-api/openapi-ts": "^0.99.0",
     "@types/node": "^24",
+    "@vitest/browser": "^4.1.12",
+    "@vitest/browser-playwright": "^4.1.12",
     "esbuild": "^0.28.1",
+    "playwright": "^1.64.1",
     "typescript": "^6",
     "vitest": "^4.1.12"
   }