@lumavox/client 0.16.0 → 0.17.0

package/README.md

@@ -191,6 +191,13 @@ allows it where the browser cancels echo (not on an iPhone or iPad);
 `assumeNoEcho` with headphones on. `loudness()` is the built-in detector,
 `detector` your own.
 
+**Half-duplex**, where echo cannot be cancelled: `listen(conversation,
+audio, { halfDuplex: true })` keeps the microphone open but sends nothing
+while the agent's voice is heard — on the player's clock, then 300 ms more
+(`afterVoiceMs`) for the room's echo — and talking never stops it, so the
+agent never hears itself and it is never refused for echo. Something said
+as the voice starts ends there as a turn, answered after the reply.
+
 ## Runtimes
 
 Browsers, React Native (Hermes, ES2020 — and each binary frame reaches JS

package/dist/audio.d.ts

@@ -34,6 +34,12 @@ export interface AgentAudio {
      */
     startCapture(onPcm: (pcm: ArrayBuffer) => void, options?: {
         rate?: number;
+        /**
+         * Hands-free asks it, once `handsFree` said yes: take the agent's
+         * voice out of the microphone — a phone's voice processing and call
+         * audio. An adapter that always does, as a browser's, ignores it.
+         */
+        echoCancellation?: boolean;
     }): Promise<{
         rate: number;
     }>;

package/dist/hands-free.d.ts

@@ -25,10 +25,36 @@
  *   cancellation leaves of its voice is louder than a quiet room.
  * - **Refused where the agent would hear itself** (D16): an adapter that
  *   says so, or cannot say (`AgentAudio.handsFree`) unless the app knows.
+ *   On its yes the recorder is asked to cancel the echo
+ *   (`startCapture`'s `echoCancellation`): a phone's is a mode of its own —
+ *   the call's audio, quieter — never wanted with headphones on or in
+ *   half-duplex.
+ * - **Half-duplex** (D19, `halfDuplex`), where there is no echo to cancel —
+ *   a phone on `react-native-audio-api` 0.13, a browser that cannot cancel
+ *   it: the microphone stays open, but **while the agent's voice is heard
+ *   nothing is sent, the detector rests and there is no barge-in**, so the
+ *   agent never hears itself and `handsFree()` is not asked. Heard is the
+ *   player's clock (`PlaybackClock`: each frame's `startsAt` and
+ *   `duration`, against `heardTime()`), never when frames are queued: a
+ *   phone queues a reply seconds ahead of its sound. An adapter with no
+ *   clock is taken at its word, from an `audio.start` to `onDrained`.
+ *   **The voice has the floor**: an utterance open when it starts ends
+ *   there as a turn — what was said before it is kept, and answered after
+ *   the reply. **Then `afterVoiceMs`, 300 ms, before the detector listens
+ *   again**: what the clock cannot know of a phone's sound path (0.13 says
+ *   none; most phones' loudspeakers take 20 to 100 ms), the microphone's own
+ *   latency (10 to 50 ms), and the room's echo falling 30 dB (half of a
+ *   furnished room's 0.3 to 0.6 s reverberation) — else the agent's last
+ *   syllable, ringing on, opens a recording of itself. It costs no reply
+ *   any time: those frames are kept as pre-roll, so words begun in it keep
+ *   their start; only a word over before it ends — a quick "yes" under
+ *   0.3 s — goes unheard.
  *
  * Time here is audio time, counted in 20 ms frames, never the clock's: a
  * busy thread delivers late, and the frames still say how long the
- * silence was.
+ * silence was. Half-duplex alone reads the player's clock, to set each
+ * frame against the voice: the last sample delivered is heard now, and
+ * each before it a sample earlier.
  */
 import type { AgentAudio } from "./audio.js";
 import type { Conversation, TurnOptions } from "./conversation.js";
@@ -88,6 +114,19 @@ export interface ListenOptions extends TurnOptions {
     wordsWithinMs?: number;
     /** The app knows the agent's voice stays out of the microphone — headphones, say — though the adapter cannot tell. */
     assumeNoEcho?: boolean;
+    /**
+     * Half-duplex: the microphone stays open, but nothing is sent while the
+     * agent's voice is heard, and talking never stops it — for where echo
+     * cannot be cancelled, such as a phone, so `handsFree()` is not asked.
+     */
+    halfDuplex?: boolean;
+    /**
+     * Half-duplex: after the agent's voice is heard ending, how long the
+     * detector still waits, ms: 300 — so the room's echo of its last word
+     * does not open a recording. Words begun in it keep their start; one
+     * over before it ends is not heard. Lower with headphones on.
+     */
+    afterVoiceMs?: number;
     /** `listening` for the person, `hearing` them, `paused` while the device has the audio, then `stopped`. */
     onState?(state: ListeningState): void;
     /** The person talked over the agent: its voice was stopped. */
@@ -107,12 +146,13 @@ export type ListenOutcome = {
     reason: string;
 };
 /** Why `listen` refuses an adapter that cannot say whether the agent hears itself. */
-export declare const NO_ECHO_ANSWER = "This audio adapter cannot say whether the agent's voice stays out of the microphone (`AgentAudio.handsFree`). Use push-to-talk, or pass `assumeNoEcho` if you know it does.";
+export declare const NO_ECHO_ANSWER = "This audio adapter cannot say whether the agent's voice stays out of the microphone (`AgentAudio.handsFree`). Use half-duplex (`halfDuplex`) or push-to-talk, or pass `assumeNoEcho` if you know it does.";
 /**
  * Leave the microphone open on `conversation`: every utterance the
- * detector hears becomes a turn, and talking over the agent stops it.
- * Answers `ok: false` with the reason, in words an app can show, where the
- * agent would hear itself or the microphone would not open — push-to-talk
- * still works there.
+ * detector hears becomes a turn, and talking over the agent stops it — or,
+ * with `halfDuplex`, nothing is heard while the agent speaks. Answers
+ * `ok: false` with the reason, in words an app can show, where the agent
+ * would hear itself or the microphone would not open — half-duplex and
+ * push-to-talk still work there.
  */
 export declare function listen(conversation: Conversation, audio: AgentAudio, options?: ListenOptions): Promise<ListenOutcome>;

package/dist/hands-free.js

@@ -40,16 +40,17 @@ export function loudness(options = {}) {
     };
 }
 /** Why `listen` refuses an adapter that cannot say whether the agent hears itself. */
-export const NO_ECHO_ANSWER = "This audio adapter cannot say whether the agent's voice stays out of the microphone (`AgentAudio.handsFree`). Use push-to-talk, or pass `assumeNoEcho` if you know it does.";
+export const NO_ECHO_ANSWER = "This audio adapter cannot say whether the agent's voice stays out of the microphone (`AgentAudio.handsFree`). Use half-duplex (`halfDuplex`) or push-to-talk, or pass `assumeNoEcho` if you know it does.";
 /**
  * Leave the microphone open on `conversation`: every utterance the
- * detector hears becomes a turn, and talking over the agent stops it.
- * Answers `ok: false` with the reason, in words an app can show, where the
- * agent would hear itself or the microphone would not open — push-to-talk
- * still works there.
+ * detector hears becomes a turn, and talking over the agent stops it — or,
+ * with `halfDuplex`, nothing is heard while the agent speaks. Answers
+ * `ok: false` with the reason, in words an app can show, where the agent
+ * would hear itself or the microphone would not open — half-duplex and
+ * push-to-talk still work there.
  */
 export async function listen(conversation, audio, options = {}) {
-    if (!options.assumeNoEcho) {
+    if (!options.assumeNoEcho && !options.halfDuplex) {
         const echo = audio.handsFree
             ? await audio.handsFree()
             : { ok: false, reason: NO_ECHO_ANSWER };
@@ -80,6 +81,12 @@ class HandsFree {
         this.playingTurn = null;
         /** The reply a barge-in stopped: its late windows are not the agent speaking. */
         this.stoppedTurn = null;
+        /** Half-duplex, on that clock: the agent's voice as scheduled, in runs of frames back to back. */
+        this.voice = [];
+        /** The clock as last read, to see it go back. */
+        this.read = 0;
+        /** Half-duplex with no clock: audio time since the voice was last playing. */
+        this.sinceVoiceMs = Number.POSITIVE_INFINITY;
         this.off = [];
         this.conversation = conversation;
         this.audio = audio;
@@ -89,12 +96,18 @@ class HandsFree {
         this.startAfterMs = options.startAfterMs ?? 100;
         this.wordsWithinMs = options.wordsWithinMs ?? 3000;
         this.ringFrames = Math.ceil(((options.preRollMs ?? 300) + this.startAfterMs) / FRAME_MS);
+        this.halfDuplex = options.halfDuplex === true;
+        this.afterVoiceMs = options.afterVoiceMs ?? 300;
+        this.clock = this.halfDuplex ? clockOf(audio) : null;
     }
     /** The microphone, and everything watched: `null`, or why it did not open. */
     async start() {
         this.off.push(this.conversation.onFrame((frame) => this.frame(frame)), this.audio.onDrained(() => {
             this.playing = false;
         }));
+        if (this.clock) {
+            this.off.push(this.clock.onScheduled((audio) => this.scheduled(audio)), this.clock.onStopped(() => this.cut()));
+        }
         const interrupted = this.audio.onInterruption?.((now) => this.interrupted(now));
         if (interrupted)
             this.off.push(interrupted);
@@ -119,7 +132,14 @@ class HandsFree {
     async capture() {
         let granted;
         try {
-            ({ rate: granted } = await this.audio.startCapture((pcm) => this.pcm(pcm), { rate: this.options.rate ?? 16000 }));
+            ({ rate: granted } = await this.audio.startCapture((pcm) => this.pcm(pcm), {
+                rate: this.options.rate ?? 16000,
+                // On the adapter's word that it can: half-duplex and the app's
+                // own word need none, and a phone's sounds better without.
+                ...(this.options.halfDuplex || this.options.assumeNoEcho
+                    ? {}
+                    : { echoCancellation: true }),
+            }));
         }
         catch (failure) {
             // swallowed: it is the answer's reason, the app's to show.
@@ -170,11 +190,16 @@ class HandsFree {
         void this.capture().then((failed) => {
             if (this.state !== "paused")
                 return;
-            // Still taken, or gone: stopped, and the app's next `listen` tries again.
-            if (failed === null)
+            if (failed === null) {
                 this.set("listening");
-            else
-                this.close();
+                return;
+            }
+            // Still taken, or gone: stopped, and the app's next `listen` tries
+            // again. Given back, so what the capture set up is undone.
+            this.close();
+            this.audio.stopCapture().catch(() => {
+                // swallowed: nothing was open; the adapter has let go either way.
+            });
         });
     }
     pcm(buffer) {
@@ -187,14 +212,25 @@ class HandsFree {
             samples.set(this.carry, 0);
             samples.set(fresh, this.carry.length);
         }
+        // Half-duplex: the last sample is heard now, on the player's clock.
+        const heard = this.clock ? this.heardNow() : 0;
         let at = 0;
         for (; at + this.frameSamples <= samples.length; at += this.frameSamples) {
-            this.step(samples.slice(at, at + this.frameSamples));
+            const end = at + this.frameSamples;
+            this.step(samples.slice(at, end), heard - (samples.length - end) / this.rate);
         }
         this.carry = samples.slice(at);
     }
-    step(frame) {
-        const voice = this.detector.push(frame, this.rate, this.playing);
+    /** One 20 ms frame; `endsAt`, half-duplex on a clock, when its last sample was heard. */
+    step(frame, endsAt) {
+        if (this.halfDuplex) {
+            const ago = this.voiceAgo(endsAt);
+            if (ago === 0 || ago < this.afterVoiceMs) {
+                this.voiceHeard(ago === 0 ? null : frame);
+                return;
+            }
+        }
+        const voice = this.detector.push(frame, this.rate, this.playing && !this.halfDuplex);
         const talking = voice === "talking";
         if (talking) {
             this.talkMs += FRAME_MS;
@@ -206,9 +242,7 @@ class HandsFree {
         }
         const open = this.open;
         if (!open) {
-            this.ring.push(frame);
-            if (this.ring.length > this.ringFrames)
-                this.ring.shift();
+            this.keep(frame);
             if (this.waitForQuiet && this.quietMs >= this.endAfterMs) {
                 this.waitForQuiet = false;
             }
@@ -228,12 +262,96 @@ class HandsFree {
             this.end();
         }
     }
+    /** A frame of the room kept for the next utterance's pre-roll. */
+    keep(frame) {
+        this.ring.push(frame);
+        if (this.ring.length > this.ringFrames)
+            this.ring.shift();
+    }
+    /**
+     * Half-duplex: ms between the agent's voice last heard and the frame
+     * ending at `endsAt` — 0 while it is heard, infinite before it ever was.
+     */
+    voiceAgo(endsAt) {
+        if (!this.clock) {
+            // Taken at the adapter's word: playing from `audio.start` to `onDrained`.
+            this.sinceVoiceMs = this.playing ? 0 : this.sinceVoiceMs + FRAME_MS;
+            return this.playing ? 0 : this.sinceVoiceMs;
+        }
+        const startsAt = endsAt - FRAME_MS / 1000;
+        let ago = Number.POSITIVE_INFINITY;
+        for (const run of this.voice) {
+            if (run.from < endsAt && run.until > startsAt)
+                return 0;
+            if (run.until <= startsAt)
+                ago = (startsAt - run.until) * 1000;
+        }
+        return ago;
+    }
+    /**
+     * Half-duplex: the agent's voice is heard, or its tail (`tail`, the
+     * frame). The voice has the floor — an utterance open ends as a turn
+     * before any of it is sent — and the detector rests; nothing of the
+     * voice is pre-roll, its tail is.
+     */
+    voiceHeard(tail) {
+        if (this.open)
+            this.end();
+        this.talkMs = 0;
+        this.quietMs = 0;
+        if (tail)
+            this.keep(tail);
+        else
+            this.ring = [];
+    }
+    /** The clock now; one that went back — a new context, the phone's for a reply at another rate — leaves the runs kept on the old one. */
+    now() {
+        const now = this.clock.now();
+        // The stop that came with it was told; the new run starts at once.
+        if (now < this.read)
+            this.voice = [];
+        this.read = now;
+        return now;
+    }
+    /** What is heard now on the clock, the runs long past forgotten. */
+    heardNow() {
+        this.now();
+        const heard = this.clock.heardTime();
+        const past = heard - (this.afterVoiceMs + 1000) / 1000;
+        while (this.voice.length > 1 && (this.voice[0]?.until ?? 0) < past) {
+            this.voice.shift();
+        }
+        return heard;
+    }
+    /** A frame of the reply, queued to start at `startsAt`: its run of the voice, or a new one after a gap. */
+    scheduled(audio) {
+        this.now();
+        const until = audio.startsAt + audio.duration;
+        const last = this.voice[this.voice.length - 1];
+        // Back to back with the one before, to a millisecond's rounding.
+        if (last &&
+            audio.startsAt >= last.from &&
+            audio.startsAt <= last.until + 0.001) {
+            last.until = Math.max(last.until, until);
+        }
+        else {
+            this.voice.push({ from: audio.startsAt, until });
+        }
+    }
+    /** The player dropped what it had queued: the voice ends where the clock is. */
+    cut() {
+        const now = this.now();
+        this.voice = this.voice.filter((run) => run.from < now);
+        for (const run of this.voice)
+            run.until = Math.min(run.until, now);
+    }
     begin() {
-        const barging = this.playing;
+        // Half-duplex never opens over the voice: no barge-in to decide.
+        const barging = this.playing && !this.halfDuplex;
         const options = this.options;
         const id = this.conversation.startAudio({
             rate: this.rate,
-            bargeIn: true,
+            bargeIn: !this.halfDuplex,
             ...(options.language ? { language: options.language } : {}),
             ...(options.model ? { model: options.model } : {}),
             ...(options.speech === undefined ? {} : { speech: options.speech }),
@@ -273,6 +391,16 @@ class HandsFree {
         this.options.onState?.(state);
     }
 }
+/** The adapter's `PlaybackClock`, when it has one: both of this SDK's do; one of an app's own need not. */
+function clockOf(audio) {
+    const clock = audio;
+    return typeof clock.now === "function" &&
+        typeof clock.heardTime === "function" &&
+        typeof clock.onScheduled === "function" &&
+        typeof clock.onStopped === "function"
+        ? audio
+        : null;
+}
 /** 16-bit little-endian PCM as samples, on a big-endian device too. */
 function samplesOf(buffer) {
     const count = buffer.byteLength >> 1;

package/dist/web-audio/web-audio.d.ts

@@ -19,7 +19,8 @@
  *   only with the context at the device's rate, which the player keeps.
  *   Every browser on an iPhone or iPad is WebKit, whose canceller fails
  *   three ways (WebKit bugs 218012, 311451, 326286): refused there — and
- *   wherever the app turned echo cancellation off.
+ *   wherever the app turned echo cancellation off. Half-duplex runs
+ *   anywhere (`listen`'s `halfDuplex`): it never asks.
  * - **The clock and the frames are the app's** (`PlaybackClock`): each
  *   frame told as it is queued, with when it starts; the stop; what the
  *   microphone captures. A handler that throws is rethrown on its own turn,
@@ -47,11 +48,11 @@ export interface WebAudioOptions {
 /** What `play` is given: the frame's rate, and what `onScheduled` passes on when it has it. */
 type Frame = Parameters<AgentAudio["play"]>[1];
 /** Why hands-free is refused on an iPhone or iPad. */
-export declare const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use push-to-talk.";
+export declare const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use half-duplex or push-to-talk.";
 /** Why it is refused in a browser that cannot cancel echo at all. */
-export declare const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use push-to-talk.";
+export declare const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use half-duplex or push-to-talk.";
 /** Why it is refused with echo cancellation turned off (`setProcessing`). */
-export declare const ECHO_OFF = "Echo cancellation is turned off, so the agent would hear itself. Turn it on, or use push-to-talk.";
+export declare const ECHO_OFF = "Echo cancellation is turned off, so the agent would hear itself. Turn it on, or use half-duplex or push-to-talk.";
 export declare class WebAudio implements AgentAudio, PlaybackClock, DeviceChoices {
     private player;
     private recorder;

package/dist/web-audio/web-audio.js

@@ -2,11 +2,11 @@ import { PcmPlayer } from "./player.js";
 /** The reply's loudness while the person may be talking over it: about -10 dB. */
 const DUCKED = 0.3;
 /** Why hands-free is refused on an iPhone or iPad. */
-export const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use push-to-talk.";
+export const IOS_ECHO = "Hands-free is not available on an iPhone or iPad browser: its echo cancellation is broken (WebKit bugs 218012, 311451 and 326286). Use half-duplex or push-to-talk.";
 /** Why it is refused in a browser that cannot cancel echo at all. */
-export const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use push-to-talk.";
+export const NO_ECHO = "This browser cannot cancel echo, so the agent would hear itself. Use half-duplex or push-to-talk.";
 /** Why it is refused with echo cancellation turned off (`setProcessing`). */
-export const ECHO_OFF = "Echo cancellation is turned off, so the agent would hear itself. Turn it on, or use push-to-talk.";
+export const ECHO_OFF = "Echo cancellation is turned off, so the agent would hear itself. Turn it on, or use half-duplex or push-to-talk.";
 export class WebAudio {
     constructor(options = {}) {
         this.player = null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lumavox/client",
-  "version": "0.16.0",
+  "version": "0.17.0",
   "description": "Talk to Lumavox agents from a browser, a React Native app or Node, with an end user's ephemeral key: conversations on the agent's socket, sessions, memory and relations. Zero dependencies.",
   "license": "MIT",
   "type": "module",