@lumavox/client 0.7.0 → 0.10.0

package/dist/web-audio/recorder.js

@@ -0,0 +1,255 @@
+/**
+ * The microphone, as the same raw PCM this socket already plays.
+ *
+ * The mirror of `player.ts`: 16-bit signed little-endian mono, at 24 kHz —
+ * the rate the socket's audio defaults to (`DEFAULT_SAMPLE_RATE`), and one
+ * of `RECORDING_RATES`.
+ *
+ * **No `MediaRecorder`, and that is the whole design.** It produces a
+ * *container* — webm/opus, whatever the browser feels like — which the socket
+ * cannot take: binary frames there are bare samples with no header, and the
+ * server would have to decode a container to find out what it was given. Web
+ * Audio hands over float samples instead, and converting those to 16-bit
+ * integers is four lines.
+ *
+ * Three things make it work in practice:
+ *
+ *  - **This context is opened at 24 kHz** rather than resampled after the
+ *    fact. Browsers accept a `sampleRate` on the constructor and do the
+ *    conversion in the audio thread, which is both better and free. (The
+ *    player's context is the device's own: `player.ts` says why.)
+ *  - **`AudioWorklet` with a `ScriptProcessor` fallback.** The worklet runs off
+ *    the main thread, so a busy React render cannot drop samples — which is
+ *    heard as a clipped word rather than as a stutter, and is exactly the kind
+ *    of fault that would be blamed on the recogniser.
+ *  - **The track is stopped on `stop()`**, not merely disconnected. A live
+ *    `MediaStreamTrack` leaves the browser's recording indicator on, and a
+ *    microphone that looks like it is still listening is a trust problem
+ *    whatever the code is doing.
+ *
+ * Loaded on the first press (`web-audio.ts`), so a page that never records
+ * never parses it.
+ */
+/** The rate this records at, whatever is asked: the socket's default. */
+export const RECORD_SAMPLE_RATE = 24000;
+/** Multiplier from the -1..1 floats Web Audio gives to 16-bit signed. 32767,
+ *  not 32768, so a full-scale sample cannot wrap to the opposite sign. */
+const FULL_SCALE = 32767;
+/** Samples per delivered chunk — about 85 ms at 24 kHz. Small enough that the
+ *  upload overlaps the talking, large enough not to be one WebSocket frame per
+ *  animation tick.
+ *
+ *  **Applied to the worklet path too.** The worklet posts every 128-sample
+ *  render quantum; one WebSocket frame a post is about 188 frames a second of
+ *  256 bytes, each a TLS record and a message the end user DO wakes for —
+ *  sixteen times as many as at this size. Batched here, on the main thread,
+ *  where the tail can be flushed synchronously when the button is released. */
+export const FRAME_SAMPLES = 2048;
+const WORKLET = `
+class Tap extends AudioWorkletProcessor {
+  process(inputs) {
+    const channel = inputs[0]?.[0];
+    // A disconnected or silent-by-design input yields an empty array rather
+    // than zeroes; forwarding that would post empty frames forever.
+    if (channel && channel.length) this.port.postMessage(channel.slice());
+    return true;
+  }
+}
+registerProcessor("pcm-tap", Tap);
+`;
+/** Float samples to 16-bit signed little-endian, which is what the wire wants. */
+export function toPcm16(samples) {
+    const out = new DataView(new ArrayBuffer(samples.length * 2));
+    for (let i = 0; i < samples.length; i++) {
+        // Clamped: a sample outside -1..1 is legal in Web Audio and wraps rather
+        // than clips once it is an integer, which is heard as a loud crack.
+        const clamped = Math.max(-1, Math.min(1, samples[i]));
+        out.setInt16(i * 2, Math.round(clamped * FULL_SCALE), true);
+    }
+    return out.buffer;
+}
+export class PcmRecorder {
+    constructor(options) {
+        /** Whether chunks may still be delivered.
+         *
+         *  **Cleared synchronously by `stop()`, and checked before every delivery.**
+         *  Disconnecting a node does not un-post the messages it has already sent:
+         *  the worklet runs on the audio thread and its `port.postMessage` calls
+         *  arrive on the main thread as ordinary tasks, so a handful land *after*
+         *  `stop()` has returned.
+         *
+         *  That is not a tidiness problem. The caller sends each chunk into an open
+         *  `user_audio` window and closes that window immediately after stopping —
+         *  so a late chunk is binary audio arriving with no window open, which the
+         *  edge answers by closing the connection. That is how it shows: hold,
+         *  release, hold, release, and the socket dies with
+         *  "Binary audio arrived with no user_audio.start before it."
+         */
+        this.live = false;
+        this.context = null;
+        this.stream = null;
+        this.node = null;
+        this.source = null;
+        /** Samples not yet delivered — see `FRAME_SAMPLES`. */
+        this.pending = [];
+        this.pendingSamples = 0;
+        this.onChunk = options.onChunk;
+        this.sampleRate = options.sampleRate ?? RECORD_SAMPLE_RATE;
+    }
+    /** True once `start()` has a live microphone and has not been stopped. */
+    get recording() {
+        return this.live;
+    }
+    /**
+     * Hz the delivered samples actually are.
+     *
+     * The context is *asked* for `sampleRate`; browsers honour it or throw, and
+     * this reads back what was granted rather than repeating the request. It is
+     * what a caller must declare on `user_audio.start` — a declared rate the
+     * samples are not is the wrong pitch and a wrong bill, behind a 200.
+     */
+    get rate() {
+        return this.context?.sampleRate ?? this.sampleRate;
+    }
+    /**
+     * Ask for the microphone and begin delivering samples.
+     *
+     * Rejects if permission is refused, which is the caller's to report — a mic
+     * button that silently does nothing after a denied prompt is the worst of the
+     * available outcomes.
+     */
+    async start() {
+        if (this.stream)
+            return;
+        this.stream = await navigator.mediaDevices.getUserMedia({
+            audio: {
+                // The browser's own cleanup, which is better than anything worth doing
+                // here and costs nothing: a recogniser fed room echo transcribes it.
+                echoCancellation: true,
+                noiseSuppression: true,
+                autoGainControl: true,
+                channelCount: 1,
+            },
+        });
+        try {
+            this.context = new AudioContext({ sampleRate: this.sampleRate });
+            // Autoplay policy suspends a context created outside a gesture. Pressing
+            // a microphone button *is* one, so this resolves immediately — it is here
+            // for the case where the button was held from a restored tab.
+            if (this.context.state === "suspended")
+                await this.context.resume();
+            this.source = this.context.createMediaStreamSource(this.stream);
+            this.node = await this.tap(this.context);
+        }
+        catch (failure) {
+            // **The track is released on the way out** (console 0.197.0).
+            // `getUserMedia` had succeeded, so a context that refused the rate or a
+            // worklet that failed to load used to leave a live microphone —
+            // indicator on — behind an error the caller reported as "could not
+            // open the microphone".
+            this.stop();
+            throw failure;
+        }
+        // **Last, after everything that can fail.** Connecting is what starts the
+        // samples flowing, so anything between it and here would be a window in
+        // which chunks arrive before the caller has been told recording began.
+        this.live = true;
+        this.source.connect(this.node);
+        // **Not connected to the destination.** Routing the microphone to the
+        // speakers is feedback, and a `ScriptProcessorNode` needs a downstream to
+        // be pulled at all — so that fallback connects to a muted gain instead.
+    }
+    /**
+     * Stop, release the microphone, and turn the browser's indicator off.
+     *
+     * Returns the **tail**: samples batched and not yet delivered when the button
+     * was released — up to one frame, ~85 ms, the end of the last word. Returned
+     * rather than delivered through `onChunk`, because the caller has by now
+     * stopped accepting chunks (`stop()` is what it calls to stop them) and
+     * decides for itself whether the tail goes on the wire before `user_audio.end`.
+     * `null` when there is nothing held.
+     */
+    stop() {
+        // **First, before anything else.** Everything below is asynchronous in
+        // effect — a disconnected node's already-posted messages still arrive — and
+        // this is the only part that takes effect immediately.
+        this.live = false;
+        const tail = this.pendingSamples ? toPcm16(this.takePending()) : null;
+        this.source?.disconnect();
+        this.node?.disconnect();
+        for (const track of this.stream?.getTracks() ?? [])
+            track.stop();
+        void this.context?.close();
+        this.source = null;
+        this.node = null;
+        this.stream = null;
+        this.context = null;
+        return tail;
+    }
+    /** Batch a block of samples, delivering whole frames as they fill. */
+    deliver(samples) {
+        if (!this.live)
+            return;
+        this.pending.push(samples);
+        this.pendingSamples += samples.length;
+        while (this.pendingSamples >= FRAME_SAMPLES) {
+            this.onChunk(toPcm16(this.takePending(FRAME_SAMPLES)));
+        }
+    }
+    /** Up to `count` pending samples (all of them by default), as one block. */
+    takePending(count = this.pendingSamples) {
+        const out = new Float32Array(Math.min(count, this.pendingSamples));
+        let filled = 0;
+        while (filled < out.length) {
+            const head = this.pending[0];
+            const take = Math.min(head.length, out.length - filled);
+            out.set(head.subarray(0, take), filled);
+            filled += take;
+            if (take === head.length)
+                this.pending.shift();
+            else
+                this.pending[0] = head.subarray(take);
+        }
+        this.pendingSamples -= out.length;
+        return out;
+    }
+    async tap(context) {
+        if (context.audioWorklet) {
+            try {
+                const url = URL.createObjectURL(new Blob([WORKLET], { type: "application/javascript" }));
+                try {
+                    await context.audioWorklet.addModule(url);
+                }
+                finally {
+                    // Revoked as soon as it is loaded: the module is compiled by then,
+                    // and a blob URL left behind is a leak for the tab's lifetime.
+                    URL.revokeObjectURL(url);
+                }
+                const node = new AudioWorkletNode(context, "pcm-tap");
+                node.port.onmessage = (event) => {
+                    this.deliver(event.data);
+                };
+                return node;
+            }
+            catch {
+                // swallowed: fall through. Worklets are unavailable over plain HTTP on
+                // some browsers and the fallback is correct, just less pleasant.
+            }
+        }
+        // `ScriptProcessorNode` is deprecated and still universally supported. It
+        // runs on the main thread, so a heavy render can drop a block — acceptable
+        // as a fallback, not as the default.
+        const node = context.createScriptProcessor(FRAME_SAMPLES, 1, 0);
+        node.onaudioprocess = (event) => {
+            // Copied: the buffer is the node's and is reused for the next block.
+            this.deliver(event.inputBuffer.getChannelData(0).slice());
+        };
+        // Silent sink: without a downstream connection the node is never pulled and
+        // `onaudioprocess` never fires, which reads as a dead microphone.
+        const sink = context.createGain();
+        sink.gain.value = 0;
+        node.connect(sink);
+        sink.connect(context.destination);
+        return node;
+    }
+}

package/dist/web-audio/web-audio.d.ts

@@ -0,0 +1,56 @@
+/**
+ * The browser's player and recorder as an `AgentAudio`: what
+ * `@lumavox/react`'s `useAgent` plays every reply through and records with
+ * — the console's since client 0.8.0, which it was written for.
+ *
+ * - **The player is made on the first click** (`prime`): every browser
+ *   suspends an `AudioContext` created outside a user gesture, and a
+ *   suspended one accepts every scheduling call silently.
+ * - **Each frame says its rate**: `audio.start` carries the window's own
+ *   (the agent's voice is re-read every turn), and the player makes each
+ *   buffer at it.
+ * - **The recorder is loaded on the first press**: `recorder.ts` holds the
+ *   AudioWorklet setup, kept out of a page that never records. It records
+ *   at 24 kHz whatever is asked — a rate the socket takes. Its tail — the
+ *   last partial frame, the end of the last word — is delivered before
+ *   `stopCapture` settles, or the transcript never gets it.
+ * - **Hands-free where the browser cancels echo** (`handsFree`): Chrome,
+ *   Edge and Firefox remove a tab's own sound from its microphone — Firefox
+ *   only with the context at the device's rate, which the player keeps.
+ *   Every browser on an iPhone or iPad is WebKit, whose canceller fails
+ *   three ways (WebKit bugs 218012, 311451, 326286): refused there.
+ */
+import type { AgentAudio } from "../audio.js";
+/** Why hands-free is refused on an iPhone or iPad. */
+export declare const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use push-to-talk.";
+/** Why it is refused in a browser that cannot cancel echo at all. */
+export declare const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use push-to-talk.";
+export declare class WebAudio implements AgentAudio {
+    private player;
+    private recorder;
+    private onPcm;
+    private readonly drained;
+    /** Make the player and unlock it. **Call it from the click**, before any `await`. */
+    prime(): Promise<void>;
+    play(pcm: ArrayBuffer, frame: {
+        sample_rate: number;
+    }): void;
+    stop(): void;
+    onDrained(fn: () => void): () => void;
+    /** At the recorder's own rate, 24 kHz, whatever is asked. */
+    startCapture(onPcm: (pcm: ArrayBuffer) => void, _options?: {
+        rate?: number;
+    }): Promise<{
+        rate: number;
+    }>;
+    stopCapture(): Promise<void>;
+    duck(on: boolean): void;
+    handsFree(): Promise<{
+        ok: true;
+    } | {
+        ok: false;
+        reason: string;
+    }>;
+    /** The conversation is over: the player's context goes too. */
+    dispose(): void;
+}

package/dist/web-audio/web-audio.js

@@ -0,0 +1,85 @@
+import { PcmPlayer } from "./player.js";
+/** The reply's loudness while the person may be talking over it: about -10 dB. */
+const DUCKED = 0.3;
+/** Why hands-free is refused on an iPhone or iPad. */
+export const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use push-to-talk.";
+/** Why it is refused in a browser that cannot cancel echo at all. */
+export const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use push-to-talk.";
+export class WebAudio {
+    constructor() {
+        this.player = null;
+        this.recorder = null;
+        this.onPcm = null;
+        this.drained = new Set();
+    }
+    /** Make the player and unlock it. **Call it from the click**, before any `await`. */
+    prime() {
+        this.player ?? (this.player = new PcmPlayer({
+            onIdle: () => {
+                for (const fn of this.drained)
+                    fn();
+            },
+        }));
+        return this.player.resume();
+    }
+    play(pcm, frame) {
+        if (!this.player)
+            void this.prime();
+        const player = this.player;
+        player.setSampleRate(frame.sample_rate);
+        player.push(pcm);
+    }
+    stop() {
+        this.player?.stop();
+    }
+    onDrained(fn) {
+        this.drained.add(fn);
+        return () => {
+            this.drained.delete(fn);
+        };
+    }
+    /** At the recorder's own rate, 24 kHz, whatever is asked. */
+    async startCapture(onPcm, _options) {
+        const { PcmRecorder } = await import("./recorder.js");
+        const mic = new PcmRecorder({ onChunk: (pcm) => this.onPcm?.(pcm) });
+        this.onPcm = onPcm;
+        this.recorder = mic;
+        try {
+            await mic.start();
+        }
+        catch (failure) {
+            if (this.recorder === mic)
+                this.recorder = null;
+            throw failure;
+        }
+        return { rate: mic.rate };
+    }
+    async stopCapture() {
+        const mic = this.recorder;
+        const deliver = this.onPcm;
+        this.recorder = null;
+        this.onPcm = null;
+        const tail = mic?.stop();
+        if (tail)
+            deliver?.(tail);
+    }
+    duck(on) {
+        this.player?.setVolume(on ? DUCKED : 1);
+    }
+    async handsFree() {
+        const devices = typeof navigator === "undefined" ? undefined : navigator.mediaDevices;
+        if (!devices?.getSupportedConstraints?.().echoCancellation) {
+            return { ok: false, reason: NO_ECHO };
+        }
+        // iPadOS says it is a Mac: a Mac with a touch screen is an iPad.
+        const ios = /iP(hone|ad|od)/.test(navigator.userAgent) ||
+            (/Macintosh/.test(navigator.userAgent) && navigator.maxTouchPoints > 1);
+        return ios ? { ok: false, reason: IOS_ECHO } : { ok: true };
+    }
+    /** The conversation is over: the player's context goes too. */
+    dispose() {
+        void this.stopCapture();
+        void this.player?.close();
+        this.player = null;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lumavox/client",
-  "version": "0.7.0",
+  "version": "0.10.0",
   "description": "Talk to Lumavox agents from a browser, a React Native app or Node, with an end user's ephemeral key: conversations on the agent's socket, sessions, memory and relations. Zero dependencies.",
   "license": "MIT",
   "type": "module",
@@ -16,6 +16,11 @@
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"
     },
+    "./web-audio": {
+      "@ino/source": "./src/web-audio/index.ts",
+      "types": "./dist/web-audio/index.d.ts",
+      "default": "./dist/web-audio/index.js"
+    },
     "./package.json": "./package.json"
   },
   "scripts": {
@@ -27,14 +32,18 @@
     "format": "biome format --write",
     "lint": "biome lint",
     "test": "vitest run",
-    "test:unit": "vitest run",
+    "test:unit": "vitest run --project unit",
+    "test:browser": "vitest run --project browser",
     "typecheck": "tsc --noEmit && tsc --noEmit -p test"
   },
   "devDependencies": {
     "@biomejs/biome": "2.6.2",
     "@hey-api/openapi-ts": "^0.99.0",
     "@types/node": "^24",
+    "@vitest/browser": "^4.1.12",
+    "@vitest/browser-playwright": "^4.1.12",
     "esbuild": "^0.28.1",
+    "playwright": "^1.64.1",
     "typescript": "^6",
     "vitest": "^4.1.12"
   }