@lumavox/client 0.7.0 → 0.10.0

package/README.md

@@ -29,7 +29,9 @@ const outcome = await conversation.send("What should I see on my first morning?"
 // { status: "done", frame } | { status: "interrupted", frame } | { status: "dropped", reason, message }
 ```
 
-React and React Native: `@lumavox/react`, hooks over this.
+React: `@lumavox/react`, hooks over this. A React Native app:
+`@lumavox/react-native`, the hooks and this with push-to-talk audio, in one
+install.
 
 The guide, with every example: the docs' **SDKs** tab, *Client SDK*.
 
@@ -126,11 +128,44 @@ const turn = await conversation.endAudio();    // a spoken turn
 const words = await conversation.endAudio({ transcribeOnly: true });
 ```
 
-The SDK captures and plays nothing itself: a recorder hands it samples, a
-player takes `onAudio`'s. `floatToPcm16`, `pcm16ToFloat`, `resample`,
-`concatPcm` and `pcmSeconds` convert between them and the wire;
+`createClient` captures and plays nothing itself: a recorder hands it
+samples, a player takes `onAudio`'s. `floatToPcm16`, `pcm16ToFloat`,
+`resample`, `concatPcm` and `pcmSeconds` convert between them and the wire;
 `AgentAudio` is the interface `@lumavox/react` drives.
 
+### Audio in a browser
+
+```ts
+import { createWebAudio } from "@lumavox/client/web-audio";
+
+const audio = createWebAudio();
+startButton.onclick = () => audio.prime(); // in the click: browsers start sound only from one
+```
+
+An `AgentAudio` on Web Audio, its own import so the client carries none of
+it (~5.5 KB minified): replies play gaplessly as they arrive, at the
+device's own rate; the microphone records at 24 kHz, with the browser's
+echo cancellation, noise suppression and gain control. Hand it to
+`@lumavox/react`'s `useAgent`, or call `play`, `startCapture` and the rest
+yourself.
+
+### Hands-free
+
+```ts
+import { listen } from "@lumavox/client";
+
+const outcome = await listen(conversation, audio); // { ok: true, listening } or { ok: false, reason }
+```
+
+The microphone left open: a detector hears each utterance (300 ms before
+it sent first), 700 ms of silence ends it as a turn — a wait added to every
+reply — and nothing is sent between utterances. Talking over the agent
+lowers its voice; its first words that are not the agent's own stop it
+(`onBargeIn`). Refused where the agent would hear itself: the web audio
+allows it where the browser cancels echo (not on an iPhone or iPad);
+`assumeNoEcho` with headphones on. `loudness()` is the built-in detector,
+`detector` your own.
+
 ## Runtimes
 
 Browsers, React Native (Hermes, ES2020 — and each binary frame reaches JS

package/dist/audio.d.ts

@@ -1,12 +1,14 @@
 /**
  * What an app plugs in to make a conversation audible and to speak in it:
- * a player and a recorder. **Types only in v1**: the SDK captures and
- * plays nothing itself, so it adds no audio library to a customer's app —
- * the console's Web Audio player and recorder are the first implementation
- * (`apps/console/src/components/audio/agent-audio.ts`), and
- * `@lumavox/react`'s `useAgent` drives one: it plays every audio frame,
- * stops on a cancel, a mute or a new turn, and records at the rate the
- * recorder was granted. Voice activity detection stays the app's.
+ * a player and a recorder. `createClient` captures and plays nothing
+ * itself, so it adds no audio to a customer's app; a browser's is the
+ * optional `@lumavox/client/web-audio` (`src/web-audio/`), a phone's
+ * `@lumavox/react-native`'s, and an app may write its own. `@lumavox/react`'s
+ * `useAgent` drives one: it plays every audio frame, stops on a cancel, a
+ * mute or a new turn, records at the rate the recorder was granted, and
+ * throws away a recording the device interrupted. Hands-free — the
+ * microphone left open, a detector saying when the person talks — is
+ * `listen` (`./hands-free.ts`), on the two optional methods below.
  */
 export interface AgentAudio {
     /** Queue one frame of a reply: 16-bit little-endian PCM, in wire order. */
@@ -35,4 +37,28 @@ export interface AgentAudio {
     stopCapture(): Promise<void>;
     /** Free everything: the conversation is over. */
     dispose(): void;
+    /**
+     * Optional: `fn(true)` when the device takes the audio away — a phone
+     * call, Siri, another app, the app leaving the screen — and `fn(false)`
+     * when it gives it back. The adapter suspends playback (a reply's frames
+     * keep queueing) and closes its recorder itself; whoever drives it throws
+     * away the recording that was open, its speaker cut off mid-sentence.
+     * Answers the unsubscribe.
+     */
+    onInterruption?(fn: (interrupted: boolean) => void): () => void;
+    /**
+     * Optional, for hands-free: whether the agent's own voice stays out of
+     * the microphone — echo cancellation, or headphones — and if not, why,
+     * in words an app can show. Without it, `listen` refuses unless the app
+     * says it knows (`assumeNoEcho`): an agent that hears itself answers
+     * itself.
+     */
+    handsFree?(): Promise<{
+        ok: true;
+    } | {
+        ok: false;
+        reason: string;
+    }>;
+    /** Optional: play the reply quieter, or as before — barge-in's first step, on sound alone. */
+    duck?(on: boolean): void;
 }

package/dist/conversation.d.ts

@@ -135,6 +135,8 @@ export interface ConversationHandlers {
     onTurnStart?: On<"turn.start">;
     onHeard?: On<"heard">;
     onHeardPartial?: On<"heard.partial">;
+    /** A `bargeIn` recording heard the person's own words: stop playing the reply. */
+    onBargeIn?: On<"barge_in">;
     onKnowledge?: On<"knowledge">;
     onMood?: On<"mood">;
     onThought?: On<"thought.delta">;
@@ -203,6 +205,13 @@ export interface StartAudioOptions extends TurnOptions {
     language?: string;
     /** A transcription model (`STT_MODELS`); the agent's own when unset. */
     model?: string;
+    /**
+     * Hands-free: a recording that may start over the agent's voice. Its
+     * first words that are not the agent's own are `onBargeIn`, and stop the
+     * running turn; the recording goes on. One with no words is
+     * `discardAudio()`'s.
+     */
+    bargeIn?: boolean;
 }
 /** What `client.connect` hands the conversation. */
 export interface Setup {
@@ -221,6 +230,7 @@ export declare class Conversation {
     private readonly speech;
     private snap;
     private listeners;
+    private watchers;
     private socket;
     /** Moves with every dial and every end: a callback from an older one is stale. */
     private generation;
@@ -294,6 +304,18 @@ export declare class Conversation {
     endAudio(options?: {
         transcribeOnly?: false;
     }): Promise<TurnOutcome>;
+    /**
+     * Throw the open recording away without a word back: nothing of it is
+     * transcribed, and a running turn goes on — hands-free's false start, a
+     * cough or the agent's own voice, heard as no words.
+     */
+    discardAudio(): void;
+    /**
+     * Every frame from now on, as `onEvent` sees them: for what is added
+     * after `connect`, such as hands-free (`listen`). Answers the
+     * unsubscribe.
+     */
+    onFrame(fn: (frame: ServerFrame) => void): () => void;
     /** Stop the turn in flight, and drop what is queued and the open recording. What was said is still billed and stored. */
     cancel(): void;
     /** Stop speaking, and let the turn finish: the rest of the reply is never spoken, nor billed. */

package/dist/conversation.js

@@ -19,6 +19,7 @@ const HANDLER = {
     "turn.start": "onTurnStart",
     heard: "onHeard",
     "heard.partial": "onHeardPartial",
+    barge_in: "onBargeIn",
     knowledge: "onKnowledge",
     mood: "onMood",
     "thought.delta": "onThought",
@@ -76,6 +77,7 @@ export class Conversation {
             queued: 0,
         };
         this.listeners = new Set();
+        this.watchers = new Set();
         this.socket = null;
         /** Moves with every dial and every end: a callback from an older one is stale. */
         this.generation = 0;
@@ -179,6 +181,8 @@ export class Conversation {
             frame.language = language;
         if (model)
             frame.model = model;
+        if (options.bargeIn)
+            frame.barge_in = true;
         if (this.socket?.ready) {
             this.socket.send(frame);
             this.said(recording);
@@ -244,6 +248,35 @@ export class Conversation {
                 this.socket?.send(frame);
         });
     }
+    /**
+     * Throw the open recording away without a word back: nothing of it is
+     * transcribed, and a running turn goes on — hands-free's false start, a
+     * cough or the agent's own voice, heard as no words.
+     */
+    discardAudio() {
+        const recording = this.open;
+        if (!recording)
+            return;
+        this.open = null;
+        recording.over = true;
+        recording.dropped = "cancelled";
+        this.emit({ caption: null });
+        // Held for `ready`, it never went out: dropped, nothing of it is sent.
+        if (!recording.held && recording.socket === this.generation) {
+            this.socket?.send({ type: "user_audio.end", discard: true });
+        }
+    }
+    /**
+     * Every frame from now on, as `onEvent` sees them: for what is added
+     * after `connect`, such as hands-free (`listen`). Answers the
+     * unsubscribe.
+     */
+    onFrame(fn) {
+        this.watchers.add(fn);
+        return () => {
+            this.watchers.delete(fn);
+        };
+    }
     /** Stop the turn in flight, and drop what is queued and the open recording. What was said is still billed and stored. */
     cancel() {
         for (const pending of this.queue.splice(0)) {
@@ -535,6 +568,8 @@ export class Conversation {
             call(this.setup.handlers[name], frame);
         }
         call(this.setup.handlers.onEvent, frame);
+        for (const watcher of this.watchers)
+            call(watcher, frame);
         if (frame.type === "ready" ||
             frame.type === "turn.done" ||
             frame.type === "interrupted" ||

package/dist/events.d.ts

@@ -58,6 +58,18 @@ export interface HeardFrame extends Frame<"heard">, InTurn {
 export interface HeardPartialFrame extends Frame<"heard.partial"> {
     text: string;
 }
+/**
+ * A `bargeIn` recording heard the person's own words — not the agent's
+ * voice leaking back: stop playing the reply. The running turn it stopped,
+ * if one still ran, ends with `interrupted`. Once per recording.
+ */
+export interface BargeInFrame extends Frame<"barge_in"> {
+    /** What was heard, as `heard.partial` said it. */
+    text: string;
+    recording_id: string | null;
+    /** The turn it stopped; `null` when none still ran, the device playing on. */
+    turn: string | null;
+}
 /** A `transcribeOnly` recording's words: part of no turn. */
 export interface TranscriptFrame extends Frame<"transcript"> {
     text: string;
@@ -252,6 +264,7 @@ export type ServerFrames = Exhaustive<{
     "turn.start": TurnStartFrame;
     heard: HeardFrame;
     "heard.partial": HeardPartialFrame;
+    barge_in: BargeInFrame;
     transcript: TranscriptFrame;
     mood: MoodFrame;
     "text.delta": TextDeltaFrame;

package/dist/hands-free.d.ts

@@ -0,0 +1,112 @@
+/**
+ * Hands-free (`docs/plans/react-native.md`, D3, D4, D15, D16): the
+ * microphone stays open, a detector says when the person talks, and each
+ * utterance is one recording on the socket, as push-to-talk's.
+ *
+ * - **Pre-roll**: what was heard just before the detector said "talking"
+ *   is sent first, or the first syllable is lost (300 ms, OpenAI's).
+ * - **Silence ends an utterance**: 700 ms of it, **added to every
+ *   hands-free reply** — the mode's one latency cost, the app's to set
+ *   (`endAfterMs`). Ours, not the transcriber's: its own end of speech
+ *   needs the microphone sent all along, the silence billed.
+ * - **Only speech leaves the device**: nothing is sent between utterances.
+ * - **Barge-in in two steps** (D15): talking while the agent's voice plays
+ *   lowers it (`AgentAudio.duck`) and opens the recording; the edge says
+ *   `barge_in` on its first words that are not the agent's own, and the
+ *   voice stops. No words within `wordsWithinMs` and it was not the person:
+ *   the recording is thrown away (`discardAudio`), the voice comes back up,
+ *   and the detector must hear silence before it opens another — so the
+ *   agent's own leftover voice, or a long cough, never loops.
+ * - **The detector's threshold rises while the agent speaks**: what echo
+ *   cancellation leaves of its voice is louder than a quiet room.
+ * - **Refused where the agent would hear itself** (D16): an adapter that
+ *   says so, or cannot say (`AgentAudio.handsFree`) unless the app knows.
+ *
+ * Time here is audio time, counted in 20 ms frames, never the clock's: a
+ * busy thread delivers late, and the frames still say how long the
+ * silence was.
+ */
+import type { AgentAudio } from "./audio.js";
+import type { Conversation, TurnOptions } from "./conversation.js";
+import type { BargeInFrame } from "./events.js";
+export type Voice = "talking" | "silent";
+/**
+ * Whether a frame of the microphone is someone talking: the built-in
+ * `loudness()`, or an app's own (D4) — Silero through
+ * `onnxruntime-react-native`, say.
+ */
+export interface Detector {
+    /**
+     * One 20 ms frame of 16-bit PCM at `rate`. `speaking`: the agent's
+     * voice is playing, so what echo cancellation leaves of it is about.
+     */
+    push(frame: Int16Array, rate: number, speaking: boolean): Voice;
+}
+export interface LoudnessOptions {
+    /** The quietest talking, dBFS: -45. */
+    minDb?: number;
+    /** How far over the room's own noise talking is, dB: 15. */
+    overNoiseDb?: number;
+    /** Added while the agent speaks, dB: 10. */
+    whileSpeakingDb?: number;
+}
+/**
+ * The built-in detector: loudness, not a model — so it never stops the
+ * agent on its own (D15). Talking is a frame louder than the room's noise
+ * by `overNoiseDb`, and never quieter than `minDb`. The noise is the
+ * quietest 100 ms of the last 3 s: the gaps between words keep it low
+ * while someone talks, and a fan that starts is learnt within 3 s.
+ */
+export declare function loudness(options?: LoudnessOptions): Detector;
+export interface ListenOptions extends TurnOptions {
+    /** The detector: `loudness()` by default. */
+    detector?: Detector;
+    /** Hz to ask the recorder for: 16,000. It must grant one of `RECORDING_RATES`. */
+    rate?: number;
+    /** A language hint for every utterance, 2 to 8 characters. */
+    language?: string;
+    /** A transcription model (`STT_MODELS`); the agent's own when unset. */
+    model?: string;
+    /** Silence that ends an utterance, ms: 700 — added to every reply. */
+    endAfterMs?: number;
+    /** Sent from before the talking was heard, ms: 300. */
+    preRollMs?: number;
+    /** Talking this long opens a recording, ms: 100 — a click or a tap is shorter. */
+    startAfterMs?: number;
+    /**
+     * Over the agent's voice: no words this long into the talking, and it
+     * was not the person, ms: 1,200 — the plan's ~0.7 s and the partial
+     * transcripts' lag behind speech (left-overs #101), until a device
+     * measures it.
+     */
+    wordsWithinMs?: number;
+    /** The app knows the agent's voice stays out of the microphone — headphones, say — though the adapter cannot tell. */
+    assumeNoEcho?: boolean;
+    /** `listening` for the person, `hearing` them, `paused` while the device has the audio, then `stopped`. */
+    onState?(state: ListeningState): void;
+    /** The person talked over the agent: its voice was stopped. */
+    onBargeIn?(frame: BargeInFrame): void;
+}
+export type ListeningState = "listening" | "hearing" | "paused" | "stopped";
+export interface Listening {
+    readonly state: ListeningState;
+    /** Close the microphone; an utterance half heard is thrown away. */
+    stop(): Promise<void>;
+}
+export type ListenOutcome = {
+    ok: true;
+    listening: Listening;
+} | {
+    ok: false;
+    reason: string;
+};
+/** Why `listen` refuses an adapter that cannot say whether the agent hears itself. */
+export declare const NO_ECHO_ANSWER = "This audio adapter cannot say whether the agent's voice stays out of the microphone (`AgentAudio.handsFree`). Use push-to-talk, or pass `assumeNoEcho` if you know it does.";
+/**
+ * Leave the microphone open on `conversation`: every utterance the
+ * detector hears becomes a turn, and talking over the agent stops it.
+ * Answers `ok: false` with the reason, in words an app can show, where the
+ * agent would hear itself or the microphone would not open — push-to-talk
+ * still works there.
+ */
+export declare function listen(conversation: Conversation, audio: AgentAudio, options?: ListenOptions): Promise<ListenOutcome>;

package/dist/hands-free.js

@@ -0,0 +1,285 @@
+import { LITTLE_ENDIAN } from "./pcm.js";
+import { RECORDING_RATES } from "./protocol.gen.js";
+/** One frame's length: every rate the socket takes is a whole number of samples in it. */
+const FRAME_MS = 20;
+/**
+ * The built-in detector: loudness, not a model — so it never stops the
+ * agent on its own (D15). Talking is a frame louder than the room's noise
+ * by `overNoiseDb`, and never quieter than `minDb`. The noise is the
+ * quietest 100 ms of the last 3 s: the gaps between words keep it low
+ * while someone talks, and a fan that starts is learnt within 3 s.
+ */
+export function loudness(options = {}) {
+    const minDb = options.minDb ?? -45;
+    const over = options.overNoiseDb ?? 15;
+    const extra = options.whileSpeakingDb ?? 10;
+    const blocks = [];
+    let quietest = Number.POSITIVE_INFINITY;
+    let frames = 0;
+    return {
+        push(frame, _rate, speaking) {
+            let sum = 0;
+            for (let i = 0; i < frame.length; i++) {
+                const sample = frame[i];
+                sum += sample * sample;
+            }
+            const rms = Math.sqrt(sum / Math.max(1, frame.length)) / 32768;
+            const db = 20 * Math.log10(rms + 1e-9);
+            quietest = Math.min(quietest, db);
+            const noise = Math.min(quietest, ...blocks);
+            if (++frames === 5) {
+                blocks.push(quietest);
+                if (blocks.length > 30)
+                    blocks.shift();
+                quietest = Number.POSITIVE_INFINITY;
+                frames = 0;
+            }
+            const threshold = Math.max(minDb, noise + over) + (speaking ? extra : 0);
+            return db > threshold ? "talking" : "silent";
+        },
+    };
+}
+/** Why `listen` refuses an adapter that cannot say whether the agent hears itself. */
+export const NO_ECHO_ANSWER = "This audio adapter cannot say whether the agent's voice stays out of the microphone (`AgentAudio.handsFree`). Use push-to-talk, or pass `assumeNoEcho` if you know it does.";
+/**
+ * Leave the microphone open on `conversation`: every utterance the
+ * detector hears becomes a turn, and talking over the agent stops it.
+ * Answers `ok: false` with the reason, in words an app can show, where the
+ * agent would hear itself or the microphone would not open — push-to-talk
+ * still works there.
+ */
+export async function listen(conversation, audio, options = {}) {
+    if (!options.assumeNoEcho) {
+        const echo = audio.handsFree
+            ? await audio.handsFree()
+            : { ok: false, reason: NO_ECHO_ANSWER };
+        if (!echo.ok)
+            return echo;
+    }
+    const engine = new HandsFree(conversation, audio, options);
+    const opened = await engine.start();
+    return opened === null
+        ? { ok: true, listening: engine }
+        : { ok: false, reason: opened };
+}
+class HandsFree {
+    constructor(conversation, audio, options) {
+        this.state = "listening";
+        this.rate = 0;
+        this.frameSamples = 0;
+        this.carry = new Int16Array(0);
+        /** The last frames before an utterance: its pre-roll. */
+        this.ring = [];
+        this.talkMs = 0;
+        this.quietMs = 0;
+        /** After a false start over the agent: silence first, then another. */
+        this.waitForQuiet = false;
+        this.open = null;
+        /** The agent's voice is playing: from an `audio.start` until the adapter drains. */
+        this.playing = false;
+        this.playingTurn = null;
+        /** The reply a barge-in stopped: its late windows are not the agent speaking. */
+        this.stoppedTurn = null;
+        this.off = [];
+        this.conversation = conversation;
+        this.audio = audio;
+        this.options = options;
+        this.detector = options.detector ?? loudness();
+        this.endAfterMs = options.endAfterMs ?? 700;
+        this.startAfterMs = options.startAfterMs ?? 100;
+        this.wordsWithinMs = options.wordsWithinMs ?? 1200;
+        this.ringFrames = Math.ceil(((options.preRollMs ?? 300) + this.startAfterMs) / FRAME_MS);
+    }
+    /** The microphone, and everything watched: `null`, or why it did not open. */
+    async start() {
+        this.off.push(this.conversation.onFrame((frame) => this.frame(frame)), this.audio.onDrained(() => {
+            this.playing = false;
+        }));
+        const interrupted = this.audio.onInterruption?.((now) => this.interrupted(now));
+        if (interrupted)
+            this.off.push(interrupted);
+        const failed = await this.capture();
+        if (failed !== null)
+            this.close();
+        return failed;
+    }
+    async stop() {
+        if (this.state === "stopped")
+            return;
+        this.close();
+        await this.audio.stopCapture();
+    }
+    close() {
+        for (const off of this.off.splice(0))
+            off();
+        this.set("stopped");
+        if (this.open)
+            this.throwAway();
+    }
+    async capture() {
+        let granted;
+        try {
+            ({ rate: granted } = await this.audio.startCapture((pcm) => this.pcm(pcm), { rate: this.options.rate ?? 16000 }));
+        }
+        catch (failure) {
+            // swallowed: it is the answer's reason, the app's to show.
+            return `Could not open the microphone: ${failure instanceof Error ? failure.message : String(failure)}`;
+        }
+        if (!RECORDING_RATES.includes(granted)) {
+            await this.audio.stopCapture();
+            return `The microphone opened at ${granted} Hz, which cannot be transcribed: ${RECORDING_RATES.join(", ")}.`;
+        }
+        this.rate = granted;
+        this.frameSamples = (granted * FRAME_MS) / 1000;
+        return null;
+    }
+    frame(frame) {
+        if (frame.type === "audio.start" && frame.turn !== this.stoppedTurn) {
+            this.playing = true;
+            this.playingTurn = frame.turn;
+        }
+        else if (frame.type === "barge_in") {
+            this.bargeIn(frame);
+        }
+    }
+    /** The person's own words over the agent: its voice stops, and comes back up for the next reply. */
+    bargeIn(frame) {
+        const open = this.open;
+        if (!open || open.barged || frame.recording_id !== open.id)
+            return;
+        open.barged = true;
+        this.stoppedTurn = frame.turn ?? this.playingTurn;
+        this.playing = false;
+        this.audio.stop();
+        this.audio.duck?.(false);
+        this.options.onBargeIn?.(frame);
+    }
+    /** The device took the audio, and closed the recorder; or gave it back. */
+    interrupted(now) {
+        if (this.state === "stopped")
+            return;
+        if (now) {
+            if (this.open)
+                this.throwAway();
+            this.set("paused");
+            return;
+        }
+        void this.capture().then((failed) => {
+            if (this.state !== "paused")
+                return;
+            // Still taken, or gone: stopped, and the app's next `listen` tries again.
+            if (failed === null)
+                this.set("listening");
+            else
+                this.close();
+        });
+    }
+    pcm(buffer) {
+        if (this.state !== "listening" && this.state !== "hearing")
+            return;
+        const fresh = samplesOf(buffer);
+        let samples = fresh;
+        if (this.carry.length) {
+            samples = new Int16Array(this.carry.length + fresh.length);
+            samples.set(this.carry, 0);
+            samples.set(fresh, this.carry.length);
+        }
+        let at = 0;
+        for (; at + this.frameSamples <= samples.length; at += this.frameSamples) {
+            this.step(samples.slice(at, at + this.frameSamples));
+        }
+        this.carry = samples.slice(at);
+    }
+    step(frame) {
+        const voice = this.detector.push(frame, this.rate, this.playing);
+        const talking = voice === "talking";
+        if (talking) {
+            this.talkMs += FRAME_MS;
+            this.quietMs = 0;
+        }
+        else {
+            this.talkMs = 0;
+            this.quietMs += FRAME_MS;
+        }
+        const open = this.open;
+        if (!open) {
+            this.ring.push(frame);
+            if (this.ring.length > this.ringFrames)
+                this.ring.shift();
+            if (this.waitForQuiet && this.quietMs >= this.endAfterMs) {
+                this.waitForQuiet = false;
+            }
+            if (!this.waitForQuiet && this.talkMs >= this.startAfterMs)
+                this.begin();
+            return;
+        }
+        this.conversation.sendAudio(frame);
+        open.ms += FRAME_MS;
+        if (open.barging && !open.barged && open.ms >= this.wordsWithinMs) {
+            this.throwAway();
+            this.waitForQuiet = true;
+        }
+        else if (this.quietMs >= this.endAfterMs) {
+            this.end();
+        }
+    }
+    begin() {
+        const barging = this.playing;
+        const options = this.options;
+        const id = this.conversation.startAudio({
+            rate: this.rate,
+            bargeIn: true,
+            ...(options.language ? { language: options.language } : {}),
+            ...(options.model ? { model: options.model } : {}),
+            ...(options.speech === undefined ? {} : { speech: options.speech }),
+            ...(options.thoughts === undefined ? {} : { thoughts: options.thoughts }),
+        });
+        this.open = { id, barging, barged: false, ms: this.talkMs };
+        if (barging)
+            this.audio.duck?.(true);
+        for (const frame of this.ring.splice(0))
+            this.conversation.sendAudio(frame);
+        this.set("hearing");
+    }
+    /** Silence: a turn — but over the agent, only once the person's own words were heard. */
+    end() {
+        const open = this.open;
+        if (open.barging && !open.barged) {
+            this.throwAway();
+            return;
+        }
+        this.open = null;
+        if (open.barging)
+            this.audio.duck?.(false);
+        // Answered on the conversation's handlers; it never rejects.
+        void this.conversation.endAudio();
+        this.set("listening");
+    }
+    /** Not the person: thrown away, and the agent's voice back up. */
+    throwAway() {
+        const open = this.open;
+        this.open = null;
+        this.conversation.discardAudio();
+        if (open.barging && !open.barged)
+            this.audio.duck?.(false);
+        if (this.state === "hearing")
+            this.set("listening");
+    }
+    set(state) {
+        if (this.state === state)
+            return;
+        this.state = state;
+        this.options.onState?.(state);
+    }
+}
+/** 16-bit little-endian PCM as samples, on a big-endian device too. */
+function samplesOf(buffer) {
+    const count = buffer.byteLength >> 1;
+    if (LITTLE_ENDIAN)
+        return new Int16Array(buffer.slice(0, count * 2));
+    const view = new DataView(buffer);
+    const out = new Int16Array(count);
+    for (let i = 0; i < count; i++)
+        out[i] = view.getInt16(i * 2, true);
+    return out;
+}

package/dist/index.d.ts

@@ -13,6 +13,7 @@ export { type CloseInfo, type ConnectOptions, Conversation, type ConversationHan
 export { AuthenticationError, ConflictError, ConnectionError, InsufficientCreditsError, NotFoundError, PermissionError, PlatformError, type PlatformErrorInit, RateLimitError, ServerError, TimeoutError, ValidationError, } from "./errors.js";
 export type * from "./events.js";
 export type * from "./gen/index.js";
+export { type Detector, type Listening, type ListeningState, type ListenOptions, type ListenOutcome, type LoudnessOptions, listen, loudness, NO_ECHO_ANSWER, type Voice, } from "./hands-free.js";
 export { concatPcm, floatToPcm16, LITTLE_ENDIAN, pcm16ToFloat, pcmSeconds, resample, } from "./pcm.js";
 export * from "./protocol.gen.js";
 export { type AudioFrame, closeName, isFinalClose, MAX_RECONNECTS, type Reconnect, reconnectAfter, type Warning, type WebSocketConstructor, type WebSocketLike, } from "./socket.js";

package/dist/index.js

@@ -10,6 +10,7 @@
 export { createClient, DEFAULT_BASE_URL, } from "./client.js";
 export { Conversation, READY_DEADLINE_MS, } from "./conversation.js";
 export { AuthenticationError, ConflictError, ConnectionError, InsufficientCreditsError, NotFoundError, PermissionError, PlatformError, RateLimitError, ServerError, TimeoutError, ValidationError, } from "./errors.js";
+export { listen, loudness, NO_ECHO_ANSWER, } from "./hands-free.js";
 export { concatPcm, floatToPcm16, LITTLE_ENDIAN, pcm16ToFloat, pcmSeconds, resample, } from "./pcm.js";
 export * from "./protocol.gen.js";
 export { closeName, isFinalClose, MAX_RECONNECTS, reconnectAfter, } from "./socket.js";