@lumavox/client 0.7.0 → 0.10.0

package/dist/protocol.gen.d.ts

@@ -4,7 +4,7 @@ export declare const PROTOCOL = "ino.v2";
 export declare const CLIENT_VERBS: readonly ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping", "tool.confirmed", "tool.result"];
 export type ClientVerb = (typeof CLIENT_VERBS)[number];
 /** Every event the edge sends. */
-export declare const SERVER_EVENTS: readonly ["ready", "turn.start", "heard", "heard.partial", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
+export declare const SERVER_EVENTS: readonly ["ready", "turn.start", "heard", "heard.partial", "barge_in", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
 export type ServerEventName = (typeof SERVER_EVENTS)[number];
 /** The events that end a turn: exactly one of them ends every turn. */
 export declare const TERMINAL_EVENTS: readonly ["turn.done", "interrupted"];

package/dist/protocol.gen.js

@@ -7,7 +7,7 @@ export const PROTOCOL = "ino.v2";
 /** Every frame a client may send; the edge closes on any other (4009). */
 export const CLIENT_VERBS = ["auth", "user_message", "user_audio.start", "user_audio.end", "cancel", "mute", "ping", "tool.confirmed", "tool.result"];
 /** Every event the edge sends. */
-export const SERVER_EVENTS = ["ready", "turn.start", "heard", "heard.partial", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
+export const SERVER_EVENTS = ["ready", "turn.start", "heard", "heard.partial", "barge_in", "transcript", "mood", "text.delta", "text.chunk", "thought.delta", "tool.start", "tool.end", "tool.call", "tool.confirm", "knowledge", "audio.start", "audio.end", "muted", "relation", "usage", "turn.done", "interrupted", "error", "pong"];
 /** The events that end a turn: exactly one of them ends every turn. */
 export const TERMINAL_EVENTS = ["turn.done", "interrupted"];
 /** A `ping` of exactly this text is answered by the edge's runtime with `KEEPALIVE_PONG`, which has no `seq`. */

package/dist/web-audio/index.d.ts

@@ -0,0 +1,15 @@
+/**
+ * `@lumavox/client/web-audio`: a browser's player and recorder for a
+ * conversation, on Web Audio — an `AgentAudio` that `@lumavox/react`'s
+ * `useAgent` drives. An entry of its own, so `createClient` carries none of
+ * it (`test/dist.test.ts` holds both sizes); zero dependencies, as the rest.
+ *
+ * ```ts
+ * const audio = createWebAudio();
+ * button.onclick = () => audio.prime(); // in the click: browsers start audio only from one
+ * ```
+ */
+import { WebAudio } from "./web-audio.js";
+export type { WebAudio };
+/** A player and a recorder for one conversation. Call `prime()` from the click that starts it. */
+export declare function createWebAudio(): WebAudio;

package/dist/web-audio/index.js

@@ -0,0 +1,16 @@
+/**
+ * `@lumavox/client/web-audio`: a browser's player and recorder for a
+ * conversation, on Web Audio — an `AgentAudio` that `@lumavox/react`'s
+ * `useAgent` drives. An entry of its own, so `createClient` carries none of
+ * it (`test/dist.test.ts` holds both sizes); zero dependencies, as the rest.
+ *
+ * ```ts
+ * const audio = createWebAudio();
+ * button.onclick = () => audio.prime(); // in the click: browsers start audio only from one
+ * ```
+ */
+import { WebAudio } from "./web-audio.js";
+/** A player and a recorder for one conversation. Call `prime()` from the click that starts it. */
+export function createWebAudio() {
+    return new WebAudio();
+}

package/dist/web-audio/player.d.ts

@@ -0,0 +1,100 @@
+/**
+ * Playing raw PCM as it arrives, with no container and no `<audio>` element.
+ *
+ * **Why not `<audio>`.** That needs a container — a data URL with a MIME
+ * type — and the socket sends bare samples precisely to avoid one: base64 is
+ * 33% of every audio byte, and audio is most of a spoken turn. There is also
+ * nothing to point an `<audio>` element *at* until a chunk is complete, which
+ * would give back the latency the transport exists to win.
+ *
+ * So: Web Audio, one `AudioBufferSourceNode` per blob, each scheduled to start
+ * exactly where the last one ended. Three things make that sound continuous
+ * rather than like a stutter of clips:
+ *
+ *  - **The schedule is kept in the context's own clock**, never in wall time.
+ *    `playhead` is an absolute `AudioContext.currentTime`, so scheduling is
+ *    sample-accurate and drift-free even if the tab is busy.
+ *  - **A blob arriving late restarts from `currentTime`**, not from a playhead
+ *    in the past — otherwise it is scheduled to have already happened and is
+ *    simply dropped, which is heard as a missing sentence.
+ *  - **A tiny lead-in** (`SCHEDULE_AHEAD`) absorbs the jitter between a frame
+ *    arriving and it being decoded, at the cost of a few milliseconds.
+ *
+ * **The context runs at the device's own rate**; each buffer is made at the
+ * stream's (24 kHz unless `audio.start` says otherwise) and the browser
+ * resamples it as it plays — every browser does, as Web Audio requires.
+ * Firefox's echo canceller hears only a context at the device's rate, which
+ * hands-free will need; until the console's code became this module (client
+ * 0.8.0) the context was pinned to the stream's rate and rebuilt when it
+ * changed.
+ *
+ * Nothing here decodes: the format is 16-bit signed little-endian, so it is
+ * an `Int16Array` read straight into a `Float32Array`.
+ */
+export interface PcmPlayerOptions {
+    /** Hz of the PCM pushed, until `setSampleRate`: the socket's default. */
+    sampleRate?: number;
+    channels?: number;
+    /** Called when the queue drains — every scheduled source has finished. */
+    onIdle?: () => void;
+}
+export declare class PcmPlayer {
+    private context;
+    private gain;
+    private playhead;
+    /** A trailing byte from the previous frame, when one was split. */
+    private remainder;
+    private scheduled;
+    /** Bumped by `stop()`, so a source cut off by it does not count itself out of
+     *  a queue that has already been zeroed — `scheduled` went negative after a
+     *  barge-in until console 0.197.0, and `playing` lied from then on. */
+    private generation;
+    private sampleRate;
+    /** The reply's loudness, 0 to 1: kept across `stop()`'s new gain node. */
+    private volume;
+    private readonly channels;
+    private readonly onIdle?;
+    constructor(options?: PcmPlayerOptions);
+    /**
+     * The rate of the PCM pushed from here on: each `audio.start` carries its
+     * window's own, since an agent's voice may be edited mid-conversation.
+     * What is queued plays on at its own rate — every buffer carries it — so
+     * nothing is dropped and the context stays.
+     */
+    setSampleRate(rate: number): void;
+    /**
+     * Create the context, or resume one the browser suspended.
+     *
+     * **Must be called from a user gesture the first time.** Every browser
+     * suspends a context created outside one, and a suspended context accepts
+     * every scheduling call silently — so the symptom is a conversation that
+     * looks like it is speaking and makes no sound.
+     */
+    resume(): Promise<void>;
+    /**
+     * Play quieter, or as before: hands-free lowers the reply while the
+     * person may be talking over it. A 15 ms glide, so it does not click.
+     */
+    setVolume(level: number): void;
+    /** Whether anything is scheduled and still to play. */
+    get playing(): boolean;
+    /**
+     * Queue one blob of PCM.
+     *
+     * Ignored before {@link resume} — a caller that pushes audio into a player
+     * nobody unlocked has a gesture problem, and buffering it would hide that
+     * until the sound arrived minutes later in one burst.
+     */
+    push(pcm: ArrayBuffer): void;
+    /**
+     * Stop everything now — barge-in, or the turn being cancelled.
+     *
+     * Disconnecting the gain node rather than stopping each source: sources are
+     * fire-and-forget once scheduled and holding references to them purely to
+     * stop them is a leak waiting to happen. A fresh node is built on the next
+     * push.
+     */
+    stop(): void;
+    /** Release the audio device. Safe to call more than once. */
+    close(): Promise<void>;
+}

package/dist/web-audio/player.js

@@ -0,0 +1,201 @@
+/**
+ * Playing raw PCM as it arrives, with no container and no `<audio>` element.
+ *
+ * **Why not `<audio>`.** That needs a container — a data URL with a MIME
+ * type — and the socket sends bare samples precisely to avoid one: base64 is
+ * 33% of every audio byte, and audio is most of a spoken turn. There is also
+ * nothing to point an `<audio>` element *at* until a chunk is complete, which
+ * would give back the latency the transport exists to win.
+ *
+ * So: Web Audio, one `AudioBufferSourceNode` per blob, each scheduled to start
+ * exactly where the last one ended. Three things make that sound continuous
+ * rather than like a stutter of clips:
+ *
+ *  - **The schedule is kept in the context's own clock**, never in wall time.
+ *    `playhead` is an absolute `AudioContext.currentTime`, so scheduling is
+ *    sample-accurate and drift-free even if the tab is busy.
+ *  - **A blob arriving late restarts from `currentTime`**, not from a playhead
+ *    in the past — otherwise it is scheduled to have already happened and is
+ *    simply dropped, which is heard as a missing sentence.
+ *  - **A tiny lead-in** (`SCHEDULE_AHEAD`) absorbs the jitter between a frame
+ *    arriving and it being decoded, at the cost of a few milliseconds.
+ *
+ * **The context runs at the device's own rate**; each buffer is made at the
+ * stream's (24 kHz unless `audio.start` says otherwise) and the browser
+ * resamples it as it plays — every browser does, as Web Audio requires.
+ * Firefox's echo canceller hears only a context at the device's rate, which
+ * hands-free will need; until the console's code became this module (client
+ * 0.8.0) the context was pinned to the stream's rate and rebuilt when it
+ * changed.
+ *
+ * Nothing here decodes: the format is 16-bit signed little-endian, so it is
+ * an `Int16Array` read straight into a `Float32Array`.
+ */
+import { DEFAULT_SAMPLE_RATE } from "../protocol.gen.js";
+/** How far ahead of `currentTime` a freshly-started run begins. Enough to
+ *  absorb frame jitter, short enough not to be perceived as delay. */
+const SCHEDULE_AHEAD = 0.06;
+/** Divisor from 16-bit signed to the -1..1 floats Web Audio wants. 32768,
+ *  because the negative range is one larger than the positive one. */
+const FULL_SCALE = 32768;
+export class PcmPlayer {
+    constructor(options = {}) {
+        this.context = null;
+        this.gain = null;
+        this.playhead = 0;
+        /** A trailing byte from the previous frame, when one was split. */
+        this.remainder = new Uint8Array(0);
+        this.scheduled = 0;
+        /** Bumped by `stop()`, so a source cut off by it does not count itself out of
+         *  a queue that has already been zeroed — `scheduled` went negative after a
+         *  barge-in until console 0.197.0, and `playing` lied from then on. */
+        this.generation = 0;
+        /** The reply's loudness, 0 to 1: kept across `stop()`'s new gain node. */
+        this.volume = 1;
+        // The edge's default (the vocabulary's), until an `audio.start` says
+        // otherwise.
+        this.sampleRate = options.sampleRate ?? DEFAULT_SAMPLE_RATE;
+        this.channels = options.channels ?? 1;
+        this.onIdle = options.onIdle;
+    }
+    /**
+     * The rate of the PCM pushed from here on: each `audio.start` carries its
+     * window's own, since an agent's voice may be edited mid-conversation.
+     * What is queued plays on at its own rate — every buffer carries it — so
+     * nothing is dropped and the context stays.
+     */
+    setSampleRate(rate) {
+        if (!Number.isFinite(rate) || rate <= 0)
+            return;
+        this.sampleRate = rate;
+    }
+    /**
+     * Create the context, or resume one the browser suspended.
+     *
+     * **Must be called from a user gesture the first time.** Every browser
+     * suspends a context created outside one, and a suspended context accepts
+     * every scheduling call silently — so the symptom is a conversation that
+     * looks like it is speaking and makes no sound.
+     */
+    async resume() {
+        if (!this.context) {
+            this.context = new AudioContext();
+            this.gain = this.context.createGain();
+            this.gain.gain.value = this.volume;
+            this.gain.connect(this.context.destination);
+        }
+        if (this.context.state === "suspended")
+            await this.context.resume();
+    }
+    /**
+     * Play quieter, or as before: hands-free lowers the reply while the
+     * person may be talking over it. A 15 ms glide, so it does not click.
+     */
+    setVolume(level) {
+        this.volume = level;
+        if (this.context && this.gain) {
+            this.gain.gain.setTargetAtTime(level, this.context.currentTime, 0.015);
+        }
+    }
+    /** Whether anything is scheduled and still to play. */
+    get playing() {
+        return this.scheduled > 0;
+    }
+    /**
+     * Queue one blob of PCM.
+     *
+     * Ignored before {@link resume} — a caller that pushes audio into a player
+     * nobody unlocked has a gesture problem, and buffering it would hide that
+     * until the sound arrived minutes later in one burst.
+     */
+    push(pcm) {
+        const context = this.context;
+        if (!context)
+            return;
+        // **A split sample is carried, never dropped.** The edge sends whole
+        // samples per frame (`apps/edge/src/voice/tts.ts` holds an odd byte for
+        // the next blob), so this should never fire — but truncating instead is
+        // the bug that made ElevenLabs playback white noise, and it is worth being
+        // unable to reintroduce. Dropping the odd byte does not lose one sample,
+        // it shifts every byte after it by one and turns the rest of the frame
+        // into noise until something realigns it.
+        let bytes = new Uint8Array(pcm);
+        if (this.remainder.length) {
+            const joined = new Uint8Array(this.remainder.length + bytes.length);
+            joined.set(this.remainder, 0);
+            joined.set(bytes, this.remainder.length);
+            bytes = joined;
+        }
+        const usable = bytes.length - (bytes.length % 2);
+        this.remainder = bytes.slice(usable);
+        if (usable < 2)
+            return;
+        // Copied into its own buffer rather than viewed in place: `Int16Array`
+        // needs a 2-byte-aligned offset, and a frame joined to a carried byte has
+        // no such promise.
+        const samples = new Int16Array(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + usable));
+        // At the stream's rate, not the context's: the browser resamples it.
+        const buffer = context.createBuffer(this.channels, samples.length / this.channels, this.sampleRate);
+        for (let channel = 0; channel < this.channels; channel += 1) {
+            const target = buffer.getChannelData(channel);
+            for (let i = 0; i < target.length; i += 1)
+                target[i] = samples[i * this.channels + channel] / FULL_SCALE;
+        }
+        const source = context.createBufferSource();
+        source.buffer = buffer;
+        source.connect(this.gain);
+        // **The branch that keeps it gapless.** While audio is still queued, the
+        // next blob starts exactly where the last one ends; once the queue has run
+        // dry, the playhead is in the past and scheduling against it would mean
+        // "start before now", which the API resolves as "start immediately" for the
+        // first blob and drops for every one after it.
+        const now = context.currentTime;
+        const startAt = this.playhead > now ? this.playhead : now + SCHEDULE_AHEAD;
+        source.start(startAt);
+        this.playhead = startAt + buffer.duration;
+        this.scheduled += 1;
+        const generation = this.generation;
+        source.onended = () => {
+            if (generation !== this.generation)
+                return;
+            this.scheduled -= 1;
+            if (this.scheduled === 0)
+                this.onIdle?.();
+        };
+    }
+    /**
+     * Stop everything now — barge-in, or the turn being cancelled.
+     *
+     * Disconnecting the gain node rather than stopping each source: sources are
+     * fire-and-forget once scheduled and holding references to them purely to
+     * stop them is a leak waiting to happen. A fresh node is built on the next
+     * push.
+     */
+    stop() {
+        if (!this.context || !this.gain)
+            return;
+        this.gain.disconnect();
+        this.gain = this.context.createGain();
+        this.gain.gain.value = this.volume;
+        this.gain.connect(this.context.destination);
+        this.playhead = 0;
+        const wasPlaying = this.scheduled > 0;
+        this.scheduled = 0;
+        this.generation += 1;
+        // A half sample belonging to audio that will never play now.
+        this.remainder = new Uint8Array(0);
+        if (wasPlaying)
+            this.onIdle?.();
+    }
+    /** Release the audio device. Safe to call more than once. */
+    async close() {
+        const context = this.context;
+        this.context = null;
+        this.gain = null;
+        this.playhead = 0;
+        this.scheduled = 0;
+        this.remainder = new Uint8Array(0);
+        if (context && context.state !== "closed")
+            await context.close();
+    }
+}

package/dist/web-audio/recorder.d.ts

@@ -0,0 +1,114 @@
+/**
+ * The microphone, as the same raw PCM this socket already plays.
+ *
+ * The mirror of `player.ts`: 16-bit signed little-endian mono, at 24 kHz —
+ * the rate the socket's audio defaults to (`DEFAULT_SAMPLE_RATE`), and one
+ * of `RECORDING_RATES`.
+ *
+ * **No `MediaRecorder`, and that is the whole design.** It produces a
+ * *container* — webm/opus, whatever the browser feels like — which the socket
+ * cannot take: binary frames there are bare samples with no header, and the
+ * server would have to decode a container to find out what it was given. Web
+ * Audio hands over float samples instead, and converting those to 16-bit
+ * integers is four lines.
+ *
+ * Three things make it work in practice:
+ *
+ *  - **This context is opened at 24 kHz** rather than resampled after the
+ *    fact. Browsers accept a `sampleRate` on the constructor and do the
+ *    conversion in the audio thread, which is both better and free. (The
+ *    player's context is the device's own: `player.ts` says why.)
+ *  - **`AudioWorklet` with a `ScriptProcessor` fallback.** The worklet runs off
+ *    the main thread, so a busy React render cannot drop samples — which is
+ *    heard as a clipped word rather than as a stutter, and is exactly the kind
+ *    of fault that would be blamed on the recogniser.
+ *  - **The track is stopped on `stop()`**, not merely disconnected. A live
+ *    `MediaStreamTrack` leaves the browser's recording indicator on, and a
+ *    microphone that looks like it is still listening is a trust problem
+ *    whatever the code is doing.
+ *
+ * Loaded on the first press (`web-audio.ts`), so a page that never records
+ * never parses it.
+ */
+/** The rate this records at, whatever is asked: the socket's default. */
+export declare const RECORD_SAMPLE_RATE = 24000;
+/** Samples per delivered chunk — about 85 ms at 24 kHz. Small enough that the
+ *  upload overlaps the talking, large enough not to be one WebSocket frame per
+ *  animation tick.
+ *
+ *  **Applied to the worklet path too.** The worklet posts every 128-sample
+ *  render quantum; one WebSocket frame a post is about 188 frames a second of
+ *  256 bytes, each a TLS record and a message the end user DO wakes for —
+ *  sixteen times as many as at this size. Batched here, on the main thread,
+ *  where the tail can be flushed synchronously when the button is released. */
+export declare const FRAME_SAMPLES = 2048;
+export interface PcmRecorderOptions {
+    /** Called with each block of samples, already 16-bit little-endian. */
+    onChunk: (pcm: ArrayBuffer) => void;
+    sampleRate?: number;
+}
+/** Float samples to 16-bit signed little-endian, which is what the wire wants. */
+export declare function toPcm16(samples: Float32Array): ArrayBuffer;
+export declare class PcmRecorder {
+    /** Whether chunks may still be delivered.
+     *
+     *  **Cleared synchronously by `stop()`, and checked before every delivery.**
+     *  Disconnecting a node does not un-post the messages it has already sent:
+     *  the worklet runs on the audio thread and its `port.postMessage` calls
+     *  arrive on the main thread as ordinary tasks, so a handful land *after*
+     *  `stop()` has returned.
+     *
+     *  That is not a tidiness problem. The caller sends each chunk into an open
+     *  `user_audio` window and closes that window immediately after stopping —
+     *  so a late chunk is binary audio arriving with no window open, which the
+     *  edge answers by closing the connection. That is how it shows: hold,
+     *  release, hold, release, and the socket dies with
+     *  "Binary audio arrived with no user_audio.start before it."
+     */
+    private live;
+    private context;
+    private stream;
+    private node;
+    private source;
+    private readonly onChunk;
+    private readonly sampleRate;
+    /** Samples not yet delivered — see `FRAME_SAMPLES`. */
+    private pending;
+    private pendingSamples;
+    constructor(options: PcmRecorderOptions);
+    /** True once `start()` has a live microphone and has not been stopped. */
+    get recording(): boolean;
+    /**
+     * Hz the delivered samples actually are.
+     *
+     * The context is *asked* for `sampleRate`; browsers honour it or throw, and
+     * this reads back what was granted rather than repeating the request. It is
+     * what a caller must declare on `user_audio.start` — a declared rate the
+     * samples are not is the wrong pitch and a wrong bill, behind a 200.
+     */
+    get rate(): number;
+    /**
+     * Ask for the microphone and begin delivering samples.
+     *
+     * Rejects if permission is refused, which is the caller's to report — a mic
+     * button that silently does nothing after a denied prompt is the worst of the
+     * available outcomes.
+     */
+    start(): Promise<void>;
+    /**
+     * Stop, release the microphone, and turn the browser's indicator off.
+     *
+     * Returns the **tail**: samples batched and not yet delivered when the button
+     * was released — up to one frame, ~85 ms, the end of the last word. Returned
+     * rather than delivered through `onChunk`, because the caller has by now
+     * stopped accepting chunks (`stop()` is what it calls to stop them) and
+     * decides for itself whether the tail goes on the wire before `user_audio.end`.
+     * `null` when there is nothing held.
+     */
+    stop(): ArrayBuffer | null;
+    /** Batch a block of samples, delivering whole frames as they fill. */
+    private deliver;
+    /** Up to `count` pending samples (all of them by default), as one block. */
+    private takePending;
+    private tap;
+}