@lumavox/react-native 0.15.1 → 0.16.0

package/README.md

@@ -58,6 +58,18 @@ function Talk({ agentId }: { agentId: string }) {
   echo; on a phone it is refused until `react-native-audio-api` 1.0 does —
   `listen({ assumeNoEcho: true })` with headphones on. Background audio
   comes with 1.0 too.
+- **A clock for a lip-synced face or subtitles**, on a phone and the web:
+  `audio.onScheduled(({ pcm, sample_rate, chunk, startsAt }) => …)` tells
+  every frame as it is queued, before it plays; draw against
+  `audio.heardTime()`; `audio.onStopped` when a stop drops what was
+  queued; `audio.onCapture` for the microphone's frames.
+  `createAudio({ leadInMs: 350 })` gives a model that long to read a
+  reply's first frame (and adds it to the first word); `outputLatencyMs`
+  is what the phone's sound takes to the ear, which the library cannot
+  say — Bluetooth earbuds add about 150–250 ms.
+- On the web the app may choose the microphone, its processing and the
+  speaker (`inputs()`, `setInput`, `setProcessing`, `outputs()`,
+  `setOutputDevice`); on a phone the system does — `[]` and `false`.
 
 The guide, with the bare React Native setup: the docs' **SDKs** tab,
 *React Native*.

package/dist/audio.native.d.ts

@@ -1,3 +1,46 @@
+/**
+ * A phone's player and recorder, on `react-native-audio-api` 0.13.6: what
+ * Metro picks for `./audio` on iOS and Android (`index.ts`). With
+ * `session.ts`, the only file that touches the library.
+ *
+ * - **The reply plays through one `AudioBufferQueueSourceNode`**, started
+ *   once: each frame's PCM is a buffer enqueued behind the last, gapless,
+ *   and a queue that runs dry plays silence until the next. `clearBuffers`
+ *   stops it at once.
+ * - **The context runs at the reply's rate**, rebuilt — what is queued
+ *   dropped — when an `audio.start` brings another: 0.13.6's queue plays
+ *   every buffer at its context's rate whatever the buffer says (its
+ *   `AudioBufferBaseSourceNode`), so a 24 kHz reply in a 48 kHz context
+ *   would play an octave up. The device's own resampling does the rest.
+ * - **The agent stopped speaking when the last buffer enqueued has ended**,
+ *   counted by the ids `enqueueBuffer` answers: the queue's own
+ *   `isLastBufferInQueue` can say so just as the next frame is enqueued, and
+ *   a buffer `clearBuffers` dropped never ends.
+ * - **The clock is the context's, and the playhead ours**: the queue says
+ *   nothing of when a buffer starts, so each frame starts where the last
+ *   ends — `max(playhead, currentTime)`, as the web's player keeps it — and
+ *   is told so (`onScheduled`). A lead-in (`leadInMs`) is a buffer of
+ *   silence enqueued ahead of a frame that finds the queue dry. A new rate's
+ *   context starts its clock at 0 again, what the old one queued stopped.
+ * - **What is heard is the clock less the app's `outputLatencyMs`**:
+ *   0.13.6 says nothing of the device's latency — its source nodes'
+ *   `getLatency()` is their time-stretcher's — so the app, which can know
+ *   its earbuds are on, says it.
+ * - **The microphone records at 16 kHz**, mono, 100 ms a buffer — less to
+ *   send than 24 kHz, and a rate the socket takes. `onAudioReady` hands the
+ *   samples over JSI as floats, at the rate the device chose, which need
+ *   not be the one asked: read from the buffer and resampled (`resample`).
+ * - **An interruption** — a call, Siri, another app, the app leaving the
+ *   screen — suspends the context, so the reply's frames keep queueing and
+ *   play from where they stopped; and closes the recorder, whose audio is
+ *   thrown away (the hook discards the recording on the socket). The audio
+ *   comes back on its own when the system says it may, else on `resume()`
+ *   or the next press.
+ * - **No hands-free**: 0.13.6 cancels no echo, so a microphone left open
+ *   hears the agent (D12, D16). `handsFree` refuses until 1.0; with
+ *   headphones on, the app passes `assumeNoEcho`.
+ */
+import { type AgentAudio, type AudioDevice, type ScheduledAudio } from "@lumavox/client";
 import type { AudioOptions, CreateAudio, DeviceAudio, Output } from "./device-audio.js";
 /** Hz the microphone records at unless a rate the socket takes is asked. */
 export declare const RECORD_RATE = 16000;
@@ -5,23 +48,42 @@ export declare const RECORD_RATE = 16000;
 export declare const PHONE_ECHO = "Hands-free needs echo cancellation, which comes with react-native-audio-api 1.0. Use push-to-talk, or assumeNoEcho with headphones on.";
 export declare class PhoneAudio implements DeviceAudio {
     interrupted: boolean;
+    /** On a phone the system chooses the speaker. */
+    readonly canChooseOutput = false;
     private context;
     private queue;
     /** The ids of the buffers enqueued and not yet played. */
     private readonly playing;
+    /** Where the last buffer enqueued ends, on the context's clock. */
+    private playhead;
     /** A byte of a sample split across two frames. */
     private carry;
     private capture;
     private readonly drained;
+    private readonly stopped;
+    private readonly scheduled;
+    private readonly captured;
     private readonly interruptions;
     private readonly unwatch;
+    /** Seconds. */
+    private readonly leadIn;
+    private readonly latency;
     constructor(options?: AudioOptions);
     prime(): Promise<void>;
-    play(pcm: ArrayBuffer, frame: {
-        sample_rate: number;
-    }): void;
+    play(pcm: ArrayBuffer, frame: Parameters<AgentAudio["play"]>[1]): void;
     stop(): void;
     onDrained(fn: () => void): () => void;
+    now(): number;
+    heardTime(): number;
+    onScheduled(fn: (audio: ScheduledAudio) => void): () => void;
+    onStopped(fn: () => void): () => void;
+    onCapture(fn: (pcm: ArrayBuffer, rate: number) => void): () => void;
+    /** The system's microphone and speaker: none listed, none set. */
+    inputs(): Promise<AudioDevice[]>;
+    outputs(): Promise<AudioDevice[]>;
+    setInput(): void;
+    setProcessing(): void;
+    setOutputDevice(): Promise<boolean>;
     /** At 16 kHz, or at the rate asked when the socket takes it. */
     startCapture(onPcm: (pcm: ArrayBuffer) => void, options?: {
         rate?: number;
@@ -34,18 +96,25 @@ export declare class PhoneAudio implements DeviceAudio {
     handsFree(): ReturnType<DeviceAudio["handsFree"]>;
     onInterruption(fn: (interrupted: boolean) => void): () => void;
     dispose(): void;
+    /** Behind what is queued; the playhead moves to its end. */
+    private enqueue;
     /** The queue for a reply at `rate`: this one, or a new context's. */
     private player;
     /**
-     * The wire's PCM as floats. **A split sample is carried, never dropped**:
-     * the edge sends whole samples (`apps/edge/src/voice/tts.ts`), but an odd
-     * byte dropped shifts every byte after it and turns the rest into noise —
-     * the bug that once made the console's playback white noise.
+     * The wire's PCM in whole samples. **A split sample is carried, never
+     * dropped**: the edge sends whole samples (`apps/edge/src/voice/tts.ts`),
+     * but an odd byte dropped shifts every byte after it and turns the rest
+     * into noise — the bug that once made the console's playback white noise.
      */
-    private samples;
+    private whole;
     /** The audio was taken: the reply waits, the recording is thrown away. */
     private interrupt;
     private quiet;
+    /**
+     * Call each of the app's handlers. One that throws is rethrown on its own
+     * turn, where the app's error reporting sees it — not here, where it
+     * would stop the player or the recorder halfway.
+     */
     private tell;
 }
 export declare const createAudio: CreateAudio;

package/dist/audio.native.js

@@ -16,6 +16,16 @@
  *   counted by the ids `enqueueBuffer` answers: the queue's own
  *   `isLastBufferInQueue` can say so just as the next frame is enqueued, and
  *   a buffer `clearBuffers` dropped never ends.
+ * - **The clock is the context's, and the playhead ours**: the queue says
+ *   nothing of when a buffer starts, so each frame starts where the last
+ *   ends — `max(playhead, currentTime)`, as the web's player keeps it — and
+ *   is told so (`onScheduled`). A lead-in (`leadInMs`) is a buffer of
+ *   silence enqueued ahead of a frame that finds the queue dry. A new rate's
+ *   context starts its clock at 0 again, what the old one queued stopped.
+ * - **What is heard is the clock less the app's `outputLatencyMs`**:
+ *   0.13.6 says nothing of the device's latency — its source nodes'
+ *   `getLatency()` is their time-stretcher's — so the app, which can know
+ *   its earbuds are on, says it.
  * - **The microphone records at 16 kHz**, mono, 100 ms a buffer — less to
  *   send than 24 kHz, and a rate the socket takes. `onAudioReady` hands the
  *   samples over JSI as floats, at the rate the device chose, which need
@@ -42,15 +52,24 @@ const RECORD_SECONDS = 0.1;
 export class PhoneAudio {
     constructor(options = {}) {
         this.interrupted = false;
+        /** On a phone the system chooses the speaker. */
+        this.canChooseOutput = false;
         this.context = null;
         this.queue = null;
         /** The ids of the buffers enqueued and not yet played. */
         this.playing = new Set();
+        /** Where the last buffer enqueued ends, on the context's clock. */
+        this.playhead = 0;
         /** A byte of a sample split across two frames. */
         this.carry = null;
         this.capture = null;
         this.drained = new Set();
+        this.stopped = new Set();
+        this.scheduled = new Set();
+        this.captured = new Set();
         this.interruptions = new Set();
+        this.leadIn = (options.leadInMs ?? 0) / 1000;
+        this.latency = (options.outputLatencyMs ?? 0) / 1000;
         session.configure(options.output ?? "speaker");
         this.unwatch = session.watch(({ began, shouldResume }) => {
             if (began)
@@ -67,28 +86,80 @@ export class PhoneAudio {
         this.player(this.context?.sampleRate ?? DEFAULT_SAMPLE_RATE);
     }
     play(pcm, frame) {
-        const queue = this.player(frame.sample_rate);
-        const samples = this.samples(pcm);
-        if (samples.length === 0)
+        const rate = frame.sample_rate;
+        this.player(rate);
+        const whole = this.whole(pcm);
+        if (whole.byteLength === 0)
             return;
-        const buffer = this.context.createBuffer(1, samples.length, frame.sample_rate);
+        const context = this.context;
+        const samples = pcm16ToFloat(whole);
+        const buffer = context.createBuffer(1, samples.length, rate);
         buffer.copyToChannel(samples, 0);
-        this.playing.add(queue.enqueueBuffer(buffer));
+        // Read once the buffer is made, as near its enqueue as can be: copying
+        // a frame over JSI took 11 ms on Android's emulator, and a frame that
+        // starts at once was told as starting that long before it was queued.
+        const now = context.currentTime;
+        if (this.playhead <= now) {
+            // The queue ran dry: this frame starts a run, after the lead-in.
+            this.playhead = now;
+            // Never an empty buffer: the library throws on a length of 0.
+            const silence = Math.round(this.leadIn * rate);
+            if (silence > 0)
+                this.enqueue(context.createBuffer(1, silence, rate));
+        }
+        const startsAt = this.playhead;
+        this.enqueue(buffer);
+        this.tell(this.scheduled, {
+            pcm: whole,
+            sample_rate: rate,
+            channels: frame.channels,
+            chunk: frame.chunk,
+            turn: frame.turn,
+            startsAt,
+            duration: buffer.duration,
+        });
     }
     stop() {
         const was = this.playing.size > 0;
         this.queue?.clearBuffers();
         this.playing.clear();
+        this.playhead = 0;
         // Half a sample of audio that will never play now.
         this.carry = null;
-        if (was)
-            this.tell(this.drained);
+        if (!was)
+            return;
+        this.tell(this.drained);
+        this.tell(this.stopped);
     }
     onDrained(fn) {
-        this.drained.add(fn);
-        return () => {
-            this.drained.delete(fn);
-        };
+        return add(this.drained, fn);
+    }
+    now() {
+        return this.context?.currentTime ?? 0;
+    }
+    heardTime() {
+        return Math.max(0, this.now() - this.latency);
+    }
+    onScheduled(fn) {
+        return add(this.scheduled, fn);
+    }
+    onStopped(fn) {
+        return add(this.stopped, fn);
+    }
+    onCapture(fn) {
+        return add(this.captured, fn);
+    }
+    /** The system's microphone and speaker: none listed, none set. */
+    async inputs() {
+        return [];
+    }
+    async outputs() {
+        return [];
+    }
+    setInput() { }
+    setProcessing() { }
+    async setOutputDevice() {
+        return false;
     }
     /** At 16 kHz, or at the rate asked when the socket takes it. */
     async startCapture(onPcm, options) {
@@ -118,7 +189,9 @@ export class PhoneAudio {
             const at = buffer.sampleRate === rate
                 ? samples
                 : resample(samples, buffer.sampleRate, rate);
-            onPcm(floatToPcm16(at).buffer);
+            const pcm = floatToPcm16(at).buffer;
+            onPcm(pcm);
+            this.tell(this.captured, pcm, rate);
         });
         if (ready.status === "error")
             throw new Error(ready.message);
@@ -165,10 +238,7 @@ export class PhoneAudio {
         return { ok: false, reason: PHONE_ECHO };
     }
     onInterruption(fn) {
-        this.interruptions.add(fn);
-        return () => {
-            this.interruptions.delete(fn);
-        };
+        return add(this.interruptions, fn);
     }
     dispose() {
         this.stop();
@@ -178,6 +248,11 @@ export class PhoneAudio {
         this.context = null;
         this.queue = null;
     }
+    /** Behind what is queued; the playhead moves to its end. */
+    enqueue(buffer) {
+        this.playing.add(this.queue.enqueueBuffer(buffer));
+        this.playhead += buffer.duration;
+    }
     /** The queue for a reply at `rate`: this one, or a new context's. */
     player(rate) {
         if (this.queue && this.context?.sampleRate === rate)
@@ -207,12 +282,12 @@ export class PhoneAudio {
         return queue;
     }
     /**
-     * The wire's PCM as floats. **A split sample is carried, never dropped**:
-     * the edge sends whole samples (`apps/edge/src/voice/tts.ts`), but an odd
-     * byte dropped shifts every byte after it and turns the rest into noise —
-     * the bug that once made the console's playback white noise.
+     * The wire's PCM in whole samples. **A split sample is carried, never
+     * dropped**: the edge sends whole samples (`apps/edge/src/voice/tts.ts`),
+     * but an odd byte dropped shifts every byte after it and turns the rest
+     * into noise — the bug that once made the console's playback white noise.
      */
-    samples(pcm) {
+    whole(pcm) {
         let bytes = new Uint8Array(pcm);
         if (this.carry !== null) {
             const joined = new Uint8Array(bytes.length + 1);
@@ -225,7 +300,7 @@ export class PhoneAudio {
             this.carry = bytes[bytes.length - 1];
             bytes = bytes.slice(0, -1);
         }
-        return pcm16ToFloat(bytes.buffer);
+        return bytes.buffer;
     }
     /** The audio was taken: the reply waits, the recording is thrown away. */
     interrupt() {
@@ -252,11 +327,30 @@ export class PhoneAudio {
             // swallowed: an interruption may have stopped the engine already.
         });
     }
+    /**
+     * Call each of the app's handlers. One that throws is rethrown on its own
+     * turn, where the app's error reporting sees it — not here, where it
+     * would stop the player or the recorder halfway.
+     */
     tell(listeners, ...args) {
-        for (const fn of listeners)
-            fn(...args);
+        for (const fn of listeners) {
+            try {
+                fn(...args);
+            }
+            catch (error) {
+                setTimeout(() => {
+                    throw error;
+                });
+            }
+        }
     }
 }
+function add(listeners, fn) {
+    listeners.add(fn);
+    return () => {
+        listeners.delete(fn);
+    };
+}
 export const createAudio = (options) => new PhoneAudio(options);
 /** Ask for the microphone, if it was not granted: whether it is now. */
 export const requestMicrophone = () => session.microphone();

package/dist/audio.web.d.ts

@@ -1,4 +1,5 @@
 import type { CreateAudio } from "./device-audio.js";
+/** The lead-in, the microphone and its processing are the browser's; `output` and `outputLatencyMs` the phone's. */
 export declare const createAudio: CreateAudio;
 /** Ask for the microphone, if it was not granted: whether it is now. */
 export declare function requestMicrophone(): Promise<boolean>;

package/dist/audio.web.js

@@ -1,45 +1,36 @@
+/**
+ * A browser's player and recorder: what Metro picks for `./audio` on the
+ * web (`index.ts`), and what anything picking no platform file gets
+ * (`audio.ts`). `react-native-audio-api` has no recorder, no queue and no
+ * real permission request on the web, so this is the client's own
+ * `@lumavox/client/web-audio` — the console's — with the phone's extras
+ * answered as a browser can.
+ *
+ * - **`prime()` must run in the press that starts the conversation**:
+ *   browsers start sound only from a gesture.
+ * - **No interruptions are told**: a browser pauses a tab's audio itself.
+ * - **No earpiece**: `setOutput` does nothing; the speaker is chosen by its
+ *   id (`setOutputDevice`), where the browser lets a page.
+ * - **Hands-free where the browser cancels echo**, as the client's says
+ *   (`handsFree`), its reply lowered while the person may be talking.
+ * - **The client's adapter itself, with the phone's extras added**: every
+ *   method it has — the clock, the frames, the devices — is passed through
+ *   by being it, so none can be left out.
+ * - **The import needs Metro's package `exports`** — on by default from
+ *   React Native 0.79 and Expo SDK 53; before, an app sets
+ *   `unstable_enablePackageExports` (the docs' React Native page).
+ */
 import { createWebAudio } from "@lumavox/client/web-audio";
-class BrowserAudio {
-    constructor() {
-        this.interrupted = false;
-        this.web = createWebAudio();
-    }
-    prime() {
-        return this.web.prime();
-    }
-    play(pcm, frame) {
-        this.web.play(pcm, frame);
-    }
-    stop() {
-        this.web.stop();
-    }
-    onDrained(fn) {
-        return this.web.onDrained(fn);
-    }
-    startCapture(onPcm, options) {
-        return this.web.startCapture(onPcm, options);
-    }
-    stopCapture() {
-        return this.web.stopCapture();
-    }
-    dispose() {
-        this.web.dispose();
-    }
-    resume() {
-        return this.web.prime();
-    }
-    setOutput(_output) { }
-    handsFree() {
-        return this.web.handsFree();
-    }
-    duck(on) {
-        this.web.duck(on);
-    }
-    onInterruption(_fn) {
-        return () => { };
-    }
-}
-export const createAudio = () => new BrowserAudio();
+/** The lead-in, the microphone and its processing are the browser's; `output` and `outputLatencyMs` the phone's. */
+export const createAudio = (options) => {
+    const web = createWebAudio(options);
+    return Object.assign(web, {
+        interrupted: false,
+        resume: () => web.prime(),
+        setOutput: () => { },
+        onInterruption: () => () => { },
+    });
+};
 /** Ask for the microphone, if it was not granted: whether it is now. */
 export async function requestMicrophone() {
     try {

package/dist/device-audio.d.ts

@@ -1,10 +1,12 @@
 /**
  * What `createAudio()` answers on every platform — a phone's
  * (`audio.native.ts`) and a browser's (`audio.web.ts`) — so an app written
- * once runs on both: the `AgentAudio` that `useAgent` drives, and what an
+ * once runs on both: the `AgentAudio` that `useAgent` drives, the clock
+ * and the frames an app draws against (`PlaybackClock`), the devices a
+ * browser lets it choose (`DeviceChoices`, none on a phone), and what an
  * app does with the device itself.
  */
-import type { AgentAudio } from "@lumavox/client";
+import type { AgentAudio, DeviceChoices, PlaybackClock, Processing } from "@lumavox/client";
 /** Where the agent is heard with no headphones on. */
 export type Output = "speaker" | "earpiece";
 export interface AudioOptions {
@@ -15,8 +17,26 @@ export interface AudioOptions {
      * 1.0 lets the earpiece be chosen.
      */
     output?: Output;
+    /**
+     * Milliseconds a reply's first frame — or one after a gap — waits before
+     * it plays, for an app that reads each frame first (`onScheduled`): as
+     * much is added to the first word heard. None on a phone unless set, 60
+     * in a browser.
+     */
+    leadInMs?: number;
+    /**
+     * Milliseconds from the phone's player to the ear, which `heardTime()`
+     * takes off the clock: `react-native-audio-api` 0.13 cannot say it, so
+     * `0` unless set — Bluetooth earbuds add about 150 to 250. A browser says
+     * its own.
+     */
+    outputLatencyMs?: number;
+    /** The microphone, by its `inputs()` id, in a browser; a phone's system chooses. */
+    input?: string;
+    /** The browser's cleanup of the microphone; a phone's is the system's. */
+    processing?: Processing;
 }
-export interface DeviceAudio extends AgentAudio {
+export interface DeviceAudio extends AgentAudio, PlaybackClock, DeviceChoices {
     /**
      * Get ready to play. **On the web, call it from the press that starts
      * the conversation**: browsers start sound only from a gesture. On a

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lumavox/react-native",
-  "version": "0.15.1",
+  "version": "0.16.0",
   "description": "Voice conversations with Lumavox agents in a React Native app, on iOS, Android and the web: the client, the React hooks, and push-to-talk audio on react-native-audio-api.",
   "license": "MIT",
   "files": [
@@ -30,8 +30,8 @@
     "typecheck": "tsc --noEmit && tsc --noEmit -p test"
   },
   "dependencies": {
-    "@lumavox/client": "0.15.1",
-    "@lumavox/react": "0.15.1"
+    "@lumavox/client": "0.16.0",
+    "@lumavox/react": "0.16.0"
   },
   "peerDependencies": {
     "react": ">=18",