@lumavox/react-native 0.15.1 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -5
- package/app.plugin.js +5 -1
- package/dist/audio.native.d.ts +144 -11
- package/dist/audio.native.js +295 -41
- package/dist/audio.web.d.ts +1 -0
- package/dist/audio.web.js +32 -41
- package/dist/device-audio.d.ts +28 -4
- package/dist/session.d.ts +40 -2
- package/dist/session.js +116 -20
- package/package.json +5 -5
package/README.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Voice conversations with Lumavox agents in a React Native app, on iOS,
|
|
4
4
|
Android and the web — **one install**: the client (`@lumavox/client`), the
|
|
5
5
|
hooks (`@lumavox/react`), and the audio they lack, a player and a recorder
|
|
6
|
-
for **push-to-talk**, on
|
|
6
|
+
for **push-to-talk**, **half-duplex** and **hands-free**, on
|
|
7
7
|
[`react-native-audio-api`](https://github.com/software-mansion/react-native-audio-api)
|
|
8
8
|
on a phone and Web Audio in a browser.
|
|
9
9
|
|
|
@@ -54,10 +54,32 @@ function Talk({ agentId }: { agentId: string }) {
|
|
|
54
54
|
- A call, Siri or another app pauses the reply and throws away a recording
|
|
55
55
|
in progress — `interrupted`, `onAudioInterrupted`, `onAudioResumed`; so
|
|
56
56
|
does leaving the screen: no background audio in this release.
|
|
57
|
-
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
57
|
+
- **Half-duplex on every phone**: `agent.listen({ halfDuplex: true })` —
|
|
58
|
+
tap once and talk for the whole conversation; the microphone stays open,
|
|
59
|
+
but nothing is heard while the agent speaks, so it never hears itself
|
|
60
|
+
and needs no echo cancellation. It cannot be talked over: give a stop
|
|
61
|
+
button (`agent.cancel()`).
|
|
62
|
+
- **Hands-free** (`agent.listen()`, talking over the agent to stop it):
|
|
63
|
+
on the web where the browser cancels echo, and on a phone **in early
|
|
64
|
+
access**, on one build of the audio library, exactly:
|
|
65
|
+
`npm install react-native-audio-api@1.0.0-nightly-6fd4c68-20261102`,
|
|
66
|
+
then build the app again. While it listens the phone's sound is a
|
|
67
|
+
call's — the echo taken out, the agent a little quieter — and Bluetooth
|
|
68
|
+
earbuds' microphone is used. On 0.13 it is refused (`audio.handsFree()`
|
|
69
|
+
says why): offer half-duplex, or `listen({ assumeNoEcho: true })` with
|
|
70
|
+
headphones on. Background audio comes with the library's 1.0.
|
|
71
|
+
- **A clock for a lip-synced face or subtitles**, on a phone and the web:
|
|
72
|
+
`audio.onScheduled(({ pcm, sample_rate, chunk, startsAt }) => …)` tells
|
|
73
|
+
every frame as it is queued, before it plays; draw against
|
|
74
|
+
`audio.heardTime()`; `audio.onStopped` when a stop drops what was
|
|
75
|
+
queued; `audio.onCapture` for the microphone's frames.
|
|
76
|
+
`createAudio({ leadInMs: 350 })` gives a model that long to read a
|
|
77
|
+
reply's first frame (and adds it to the first word); `outputLatencyMs`
|
|
78
|
+
is what the phone's sound takes to the ear, which the library cannot
|
|
79
|
+
say — Bluetooth earbuds add about 150–250 ms.
|
|
80
|
+
- On the web the app may choose the microphone, its processing and the
|
|
81
|
+
speaker (`inputs()`, `setInput`, `setProcessing`, `outputs()`,
|
|
82
|
+
`setOutputDevice`); on a phone the system does — `[]` and `false`.
|
|
61
83
|
|
|
62
84
|
The guide, with the bare React Native setup: the docs' **SDKs** tab,
|
|
63
85
|
*React Native*.
|
package/app.plugin.js
CHANGED
|
@@ -68,7 +68,10 @@ function audioApiOptions(options = {}) {
|
|
|
68
68
|
androidFSStopWithTask: true,
|
|
69
69
|
// The library's own defaults: FFmpeg decodes compressed files, which a
|
|
70
70
|
// conversation's raw PCM never needs — `true` makes the app smaller.
|
|
71
|
+
// 1.0 turned the option round, FFmpeg off unless `enableFFmpeg`: each
|
|
72
|
+
// build reads its own and ignores the other's, so both say the same.
|
|
71
73
|
disableFFmpeg: options.disableFFmpeg ?? false,
|
|
74
|
+
enableFFmpeg: !(options.disableFFmpeg ?? false),
|
|
72
75
|
disableStaticExternalLibs: options.disableStaticExternalLibs ?? false,
|
|
73
76
|
};
|
|
74
77
|
}
|
|
@@ -90,7 +93,8 @@ function withLumavox(config, options, library) {
|
|
|
90
93
|
}
|
|
91
94
|
|
|
92
95
|
function loadLibrary() {
|
|
93
|
-
|
|
96
|
+
// With its extension: 1.0's `exports` names `./app.plugin.js` alone.
|
|
97
|
+
const plugin = require(`${LIBRARY}/app.plugin.js`);
|
|
94
98
|
return plugin.default ?? plugin;
|
|
95
99
|
}
|
|
96
100
|
|
package/dist/audio.native.d.ts
CHANGED
|
@@ -1,51 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A phone's player and recorder, on `react-native-audio-api` — 0.13.6, or
|
|
3
|
+
* the one 1.0 nightly hands-free runs on (`session.ts`, `PINNED_NIGHTLY`):
|
|
4
|
+
* what Metro picks for `./audio` on iOS and Android (`index.ts`). With
|
|
5
|
+
* `session.ts`, the only file that touches the library.
|
|
6
|
+
*
|
|
7
|
+
* - **The reply plays through one `AudioBufferQueueSourceNode`**, started
|
|
8
|
+
* once: each frame's PCM is a buffer enqueued behind the last, gapless,
|
|
9
|
+
* and a queue that runs dry plays silence until the next. `clearBuffers`
|
|
10
|
+
* stops it at once.
|
|
11
|
+
* - **The context runs at the reply's rate**, rebuilt — what is queued
|
|
12
|
+
* dropped — when an `audio.start` brings another: 0.13.6's queue plays
|
|
13
|
+
* every buffer at its context's rate whatever the buffer says (its
|
|
14
|
+
* `AudioBufferBaseSourceNode`), so a 24 kHz reply in a 48 kHz context
|
|
15
|
+
* would play an octave up. The device's own resampling does the rest.
|
|
16
|
+
* - **The agent stopped speaking when the last buffer enqueued has ended**,
|
|
17
|
+
* counted by the ids `enqueueBuffer` answers: the queue's own
|
|
18
|
+
* `isLastBufferInQueue` can say so just as the next frame is enqueued, and
|
|
19
|
+
* a buffer `clearBuffers` dropped never ends.
|
|
20
|
+
* - **The clock is the context's, and the playhead ours**: the queue says
|
|
21
|
+
* nothing of when a buffer starts, so each frame starts where the last
|
|
22
|
+
* ends — `max(playhead, currentTime)`, as the web's player keeps it — and
|
|
23
|
+
* is told so (`onScheduled`). A lead-in (`leadInMs`) is a buffer of
|
|
24
|
+
* silence enqueued ahead of a frame that finds the queue dry. A new rate's
|
|
25
|
+
* context starts its clock at 0 again, what the old one queued stopped.
|
|
26
|
+
* - **What is heard is the clock less the app's `outputLatencyMs`**:
|
|
27
|
+
* 0.13.6 says nothing of the device's latency — its source nodes'
|
|
28
|
+
* `getLatency()` is their time-stretcher's — so the app, which can know
|
|
29
|
+
* its earbuds are on, says it. Unset on the nightly, the context's own
|
|
30
|
+
* (`outputLatency` and `baseLatency`, 1.0's).
|
|
31
|
+
* - **The microphone records at 16 kHz**, mono, 100 ms a buffer — less to
|
|
32
|
+
* send than 24 kHz, and a rate the socket takes. `onAudioReady` hands the
|
|
33
|
+
* samples over JSI as floats, at the rate the device chose, which need
|
|
34
|
+
* not be the one asked: read from the buffer and resampled (`resample`).
|
|
35
|
+
* - **An interruption** — a call, Siri, another app, the app leaving the
|
|
36
|
+
* screen — suspends the context, so the reply's frames keep queueing and
|
|
37
|
+
* play from where they stopped; and closes the recorder, whose audio is
|
|
38
|
+
* thrown away (the hook discards the recording on the socket). The audio
|
|
39
|
+
* comes back on its own when the system says it may, else on `resume()`
|
|
40
|
+
* or the next press.
|
|
41
|
+
* - **Half-duplex on any build** (`listen`'s `halfDuplex`): the recorder
|
|
42
|
+
* started once for the whole conversation, which suits the library —
|
|
43
|
+
* on iOS, 0.13.6's recorder shares the player's one engine, and each
|
|
44
|
+
* start stops and starts it and each stop rebuilds it, a gap in the
|
|
45
|
+
* voice; the loudspeaker holds (`defaultToSpeaker`), and Android's
|
|
46
|
+
* recorder and player are two streams that open side by side.
|
|
47
|
+
* - **Hands-free on the pinned nightly alone** (D17, D18; `handsFree`):
|
|
48
|
+
* 0.13.6 cancels no echo, so a microphone left open hears the agent
|
|
49
|
+
* (D12, D16) — refused there, `assumeNoEcho` with headphones on. On the
|
|
50
|
+
* nightly a capture that asks for echo cancellation (`listen` does, on
|
|
51
|
+
* `handsFree`'s yes) is iOS's voice processing in `videoChat`, and
|
|
52
|
+
* Android's voice-communication input with the reply on the call's path
|
|
53
|
+
* (`androidOutputProfile`), in its communication mode
|
|
54
|
+
* (`session.configure`): its canceller hears only that path. The reply
|
|
55
|
+
* moves to it — a new context — once nothing is playing, never mid-word,
|
|
56
|
+
* and back once the capture ends. **A recorder that will not start that
|
|
57
|
+
* way fails the capture**, and `listen` answers why: the app offers
|
|
58
|
+
* half-duplex (D16, D19). Whether an Android phone's canceller works is
|
|
59
|
+
* not asked: the platform says it only to native code (left-overs #177).
|
|
60
|
+
* - **`duck`**: the reply goes through a gain, lowered while it may be
|
|
61
|
+
* talked over — barge-in's first step.
|
|
62
|
+
*/
|
|
63
|
+
import { type AgentAudio, type AudioDevice, type ScheduledAudio } from "@lumavox/client";
|
|
1
64
|
import type { AudioOptions, CreateAudio, DeviceAudio, Output } from "./device-audio.js";
|
|
2
65
|
/** Hz the microphone records at unless a rate the socket takes is asked. */
|
|
3
66
|
export declare const RECORD_RATE = 16000;
|
|
4
|
-
/** Why `handsFree` refuses on
|
|
5
|
-
export declare const PHONE_ECHO = "Hands-free needs echo cancellation, which comes with react-native-audio-api 1.0. Use push-to-talk, or assumeNoEcho with headphones on.";
|
|
67
|
+
/** Why `handsFree` refuses on 0.13. */
|
|
68
|
+
export declare const PHONE_ECHO = "Hands-free needs echo cancellation, which comes with react-native-audio-api 1.0. Use half-duplex or push-to-talk, or assumeNoEcho with headphones on.";
|
|
69
|
+
/** Why it refuses on a 1.0 build that is not the one it was tested on. */
|
|
70
|
+
export declare const untestedBuild: (version: string) => string;
|
|
6
71
|
export declare class PhoneAudio implements DeviceAudio {
|
|
7
72
|
interrupted: boolean;
|
|
73
|
+
/** On a phone the system chooses the speaker. */
|
|
74
|
+
readonly canChooseOutput = false;
|
|
8
75
|
private context;
|
|
9
76
|
private queue;
|
|
77
|
+
/** Between the queue and the speaker: the reply lowered while it may be talked over. */
|
|
78
|
+
private gain;
|
|
79
|
+
private volume;
|
|
80
|
+
/** The context's output on Android: the call's path while hands-free cancels echo. */
|
|
81
|
+
private profile;
|
|
82
|
+
/** A capture is cancelling echo: the session is hands-free's. */
|
|
83
|
+
private cancelling;
|
|
84
|
+
private unroute;
|
|
85
|
+
private output;
|
|
86
|
+
private disposed;
|
|
87
|
+
/** The call's route last asked on Android (`session.route`). */
|
|
88
|
+
private routed;
|
|
89
|
+
/**
|
|
90
|
+
* Until when an interruption is the session made again — Android's focus
|
|
91
|
+
* given up and taken back, told a moment later on its own thread — and
|
|
92
|
+
* not another app's.
|
|
93
|
+
*/
|
|
94
|
+
private settling;
|
|
10
95
|
/** The ids of the buffers enqueued and not yet played. */
|
|
11
96
|
private readonly playing;
|
|
97
|
+
/** Where the last buffer enqueued ends, on the context's clock. */
|
|
98
|
+
private playhead;
|
|
12
99
|
/** A byte of a sample split across two frames. */
|
|
13
100
|
private carry;
|
|
14
101
|
private capture;
|
|
15
102
|
private readonly drained;
|
|
103
|
+
private readonly stopped;
|
|
104
|
+
private readonly scheduled;
|
|
105
|
+
private readonly captured;
|
|
16
106
|
private readonly interruptions;
|
|
17
107
|
private readonly unwatch;
|
|
108
|
+
/** Seconds. */
|
|
109
|
+
private readonly leadIn;
|
|
110
|
+
/** The app's, or `null`: the context's own where it says one. */
|
|
111
|
+
private readonly latency;
|
|
18
112
|
constructor(options?: AudioOptions);
|
|
19
113
|
prime(): Promise<void>;
|
|
20
|
-
play(pcm: ArrayBuffer, frame:
|
|
21
|
-
sample_rate: number;
|
|
22
|
-
}): void;
|
|
114
|
+
play(pcm: ArrayBuffer, frame: Parameters<AgentAudio["play"]>[1]): void;
|
|
23
115
|
stop(): void;
|
|
24
116
|
onDrained(fn: () => void): () => void;
|
|
25
|
-
|
|
117
|
+
now(): number;
|
|
118
|
+
heardTime(): number;
|
|
119
|
+
onScheduled(fn: (audio: ScheduledAudio) => void): () => void;
|
|
120
|
+
onStopped(fn: () => void): () => void;
|
|
121
|
+
onCapture(fn: (pcm: ArrayBuffer, rate: number) => void): () => void;
|
|
122
|
+
/** The system's microphone and speaker: none listed, none set. */
|
|
123
|
+
inputs(): Promise<AudioDevice[]>;
|
|
124
|
+
outputs(): Promise<AudioDevice[]>;
|
|
125
|
+
setInput(): void;
|
|
126
|
+
setProcessing(): void;
|
|
127
|
+
setOutputDevice(): Promise<boolean>;
|
|
128
|
+
/**
|
|
129
|
+
* At 16 kHz, or at the rate asked when the socket takes it; cancelling
|
|
130
|
+
* echo when hands-free asks and the library can.
|
|
131
|
+
*/
|
|
26
132
|
startCapture(onPcm: (pcm: ArrayBuffer) => void, options?: {
|
|
27
133
|
rate?: number;
|
|
134
|
+
echoCancellation?: boolean;
|
|
28
135
|
}): Promise<{
|
|
29
136
|
rate: number;
|
|
30
137
|
}>;
|
|
138
|
+
/** The recorder, open: its echo cancelled when `cancel`. */
|
|
139
|
+
private record;
|
|
31
140
|
stopCapture(): Promise<void>;
|
|
32
141
|
resume(): Promise<void>;
|
|
33
142
|
setOutput(output: Output): void;
|
|
143
|
+
/** Hands-free's first step on sound over the reply: lowered, until the words say who it was. */
|
|
144
|
+
duck(on: boolean): void;
|
|
34
145
|
handsFree(): ReturnType<DeviceAudio["handsFree"]>;
|
|
35
146
|
onInterruption(fn: (interrupted: boolean) => void): () => void;
|
|
36
147
|
dispose(): void;
|
|
148
|
+
/** Behind what is queued; the playhead moves to its end. */
|
|
149
|
+
private enqueue;
|
|
150
|
+
/**
|
|
151
|
+
* The session hands-free's, Android's reply on the call's path and the
|
|
152
|
+
* call's route: before the recorder starts, so the canceller hears the
|
|
153
|
+
* first reply too.
|
|
154
|
+
*/
|
|
155
|
+
private enterHandsFree;
|
|
156
|
+
/** Back to push-to-talk's session; the reply leaves the call's path once it has played out. */
|
|
157
|
+
private leaveHandsFree;
|
|
158
|
+
/**
|
|
159
|
+
* The session for hands-free or not — disposed, the mode given back and
|
|
160
|
+
* nothing taken again for a screen gone — its focus's comings and goings
|
|
161
|
+
* no interruption.
|
|
162
|
+
*/
|
|
163
|
+
private reconfigure;
|
|
164
|
+
private reroute;
|
|
37
165
|
/** The queue for a reply at `rate`: this one, or a new context's. */
|
|
38
166
|
private player;
|
|
39
167
|
/**
|
|
40
|
-
* The wire's PCM
|
|
41
|
-
* the edge sends whole samples (`apps/edge/src/voice/tts.ts`),
|
|
42
|
-
* byte dropped shifts every byte after it and turns the rest
|
|
43
|
-
* the bug that once made the console's playback white noise.
|
|
168
|
+
* The wire's PCM in whole samples. **A split sample is carried, never
|
|
169
|
+
* dropped**: the edge sends whole samples (`apps/edge/src/voice/tts.ts`),
|
|
170
|
+
* but an odd byte dropped shifts every byte after it and turns the rest
|
|
171
|
+
* into noise — the bug that once made the console's playback white noise.
|
|
44
172
|
*/
|
|
45
|
-
private
|
|
173
|
+
private whole;
|
|
46
174
|
/** The audio was taken: the reply waits, the recording is thrown away. */
|
|
47
175
|
private interrupt;
|
|
48
176
|
private quiet;
|
|
177
|
+
/**
|
|
178
|
+
* Call each of the app's handlers. One that throws is rethrown on its own
|
|
179
|
+
* turn, where the app's error reporting sees it — not here, where it
|
|
180
|
+
* would stop the player or the recorder halfway.
|
|
181
|
+
*/
|
|
49
182
|
private tell;
|
|
50
183
|
}
|
|
51
184
|
export declare const createAudio: CreateAudio;
|
package/dist/audio.native.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* A phone's player and recorder, on `react-native-audio-api` 0.13.6
|
|
3
|
-
*
|
|
2
|
+
* A phone's player and recorder, on `react-native-audio-api` — 0.13.6, or
|
|
3
|
+
* the one 1.0 nightly hands-free runs on (`session.ts`, `PINNED_NIGHTLY`):
|
|
4
|
+
* what Metro picks for `./audio` on iOS and Android (`index.ts`). With
|
|
4
5
|
* `session.ts`, the only file that touches the library.
|
|
5
6
|
*
|
|
6
7
|
* - **The reply plays through one `AudioBufferQueueSourceNode`**, started
|
|
@@ -16,6 +17,17 @@
|
|
|
16
17
|
* counted by the ids `enqueueBuffer` answers: the queue's own
|
|
17
18
|
* `isLastBufferInQueue` can say so just as the next frame is enqueued, and
|
|
18
19
|
* a buffer `clearBuffers` dropped never ends.
|
|
20
|
+
* - **The clock is the context's, and the playhead ours**: the queue says
|
|
21
|
+
* nothing of when a buffer starts, so each frame starts where the last
|
|
22
|
+
* ends — `max(playhead, currentTime)`, as the web's player keeps it — and
|
|
23
|
+
* is told so (`onScheduled`). A lead-in (`leadInMs`) is a buffer of
|
|
24
|
+
* silence enqueued ahead of a frame that finds the queue dry. A new rate's
|
|
25
|
+
* context starts its clock at 0 again, what the old one queued stopped.
|
|
26
|
+
* - **What is heard is the clock less the app's `outputLatencyMs`**:
|
|
27
|
+
* 0.13.6 says nothing of the device's latency — its source nodes'
|
|
28
|
+
* `getLatency()` is their time-stretcher's — so the app, which can know
|
|
29
|
+
* its earbuds are on, says it. Unset on the nightly, the context's own
|
|
30
|
+
* (`outputLatency` and `baseLatency`, 1.0's).
|
|
19
31
|
* - **The microphone records at 16 kHz**, mono, 100 ms a buffer — less to
|
|
20
32
|
* send than 24 kHz, and a rate the socket takes. `onAudioReady` hands the
|
|
21
33
|
* samples over JSI as floats, at the rate the device chose, which need
|
|
@@ -26,33 +38,91 @@
|
|
|
26
38
|
* thrown away (the hook discards the recording on the socket). The audio
|
|
27
39
|
* comes back on its own when the system says it may, else on `resume()`
|
|
28
40
|
* or the next press.
|
|
29
|
-
* - **
|
|
30
|
-
*
|
|
31
|
-
*
|
|
41
|
+
* - **Half-duplex on any build** (`listen`'s `halfDuplex`): the recorder
|
|
42
|
+
* started once for the whole conversation, which suits the library —
|
|
43
|
+
* on iOS, 0.13.6's recorder shares the player's one engine, and each
|
|
44
|
+
* start stops and starts it and each stop rebuilds it, a gap in the
|
|
45
|
+
* voice; the loudspeaker holds (`defaultToSpeaker`), and Android's
|
|
46
|
+
* recorder and player are two streams that open side by side.
|
|
47
|
+
* - **Hands-free on the pinned nightly alone** (D17, D18; `handsFree`):
|
|
48
|
+
* 0.13.6 cancels no echo, so a microphone left open hears the agent
|
|
49
|
+
* (D12, D16) — refused there, `assumeNoEcho` with headphones on. On the
|
|
50
|
+
* nightly a capture that asks for echo cancellation (`listen` does, on
|
|
51
|
+
* `handsFree`'s yes) is iOS's voice processing in `videoChat`, and
|
|
52
|
+
* Android's voice-communication input with the reply on the call's path
|
|
53
|
+
* (`androidOutputProfile`), in its communication mode
|
|
54
|
+
* (`session.configure`): its canceller hears only that path. The reply
|
|
55
|
+
* moves to it — a new context — once nothing is playing, never mid-word,
|
|
56
|
+
* and back once the capture ends. **A recorder that will not start that
|
|
57
|
+
* way fails the capture**, and `listen` answers why: the app offers
|
|
58
|
+
* half-duplex (D16, D19). Whether an Android phone's canceller works is
|
|
59
|
+
* not asked: the platform says it only to native code (left-overs #177).
|
|
60
|
+
* - **`duck`**: the reply goes through a gain, lowered while it may be
|
|
61
|
+
* talked over — barge-in's first step.
|
|
32
62
|
*/
|
|
33
63
|
import { DEFAULT_SAMPLE_RATE, floatToPcm16, pcm16ToFloat, RECORDING_RATES, resample, } from "@lumavox/client";
|
|
34
64
|
import { AudioContext, AudioRecorder, } from "react-native-audio-api";
|
|
35
65
|
import * as session from "./session.js";
|
|
36
66
|
/** Hz the microphone records at unless a rate the socket takes is asked. */
|
|
37
67
|
export const RECORD_RATE = 16000;
|
|
38
|
-
/** Why `handsFree` refuses on
|
|
39
|
-
export const PHONE_ECHO = "Hands-free needs echo cancellation, which comes with react-native-audio-api 1.0. Use push-to-talk, or assumeNoEcho with headphones on.";
|
|
68
|
+
/** Why `handsFree` refuses on 0.13. */
|
|
69
|
+
export const PHONE_ECHO = "Hands-free needs echo cancellation, which comes with react-native-audio-api 1.0. Use half-duplex or push-to-talk, or assumeNoEcho with headphones on.";
|
|
70
|
+
/** Why it refuses on a 1.0 build that is not the one it was tested on. */
|
|
71
|
+
export const untestedBuild = (version) => `Hands-free runs on react-native-audio-api ${session.PINNED_NIGHTLY}, the build it was tested on; this app has ${version}. Use half-duplex or push-to-talk.`;
|
|
72
|
+
/** After the session is made again, how long a focus change is still its own: Android tells it on another thread, a moment later. */
|
|
73
|
+
const SETTLE_MS = 500;
|
|
74
|
+
/** The reply's loudness while the person may be talking over it (`duck`): about -10 dB, as the web's. */
|
|
75
|
+
const DUCKED = 0.3;
|
|
40
76
|
/** Seconds of microphone per buffer delivered: the recorder's own latency. */
|
|
41
77
|
const RECORD_SECONDS = 0.1;
|
|
42
78
|
export class PhoneAudio {
|
|
43
79
|
constructor(options = {}) {
|
|
44
80
|
this.interrupted = false;
|
|
81
|
+
/** On a phone the system chooses the speaker. */
|
|
82
|
+
this.canChooseOutput = false;
|
|
45
83
|
this.context = null;
|
|
46
84
|
this.queue = null;
|
|
85
|
+
/** Between the queue and the speaker: the reply lowered while it may be talked over. */
|
|
86
|
+
this.gain = null;
|
|
87
|
+
this.volume = 1;
|
|
88
|
+
/** The context's output on Android: the call's path while hands-free cancels echo. */
|
|
89
|
+
this.profile = "media";
|
|
90
|
+
/** A capture is cancelling echo: the session is hands-free's. */
|
|
91
|
+
this.cancelling = false;
|
|
92
|
+
this.unroute = null;
|
|
93
|
+
this.disposed = false;
|
|
94
|
+
/** The call's route last asked on Android (`session.route`). */
|
|
95
|
+
this.routed = null;
|
|
96
|
+
/**
|
|
97
|
+
* Until when an interruption is the session made again — Android's focus
|
|
98
|
+
* given up and taken back, told a moment later on its own thread — and
|
|
99
|
+
* not another app's.
|
|
100
|
+
*/
|
|
101
|
+
this.settling = 0;
|
|
47
102
|
/** The ids of the buffers enqueued and not yet played. */
|
|
48
103
|
this.playing = new Set();
|
|
104
|
+
/** Where the last buffer enqueued ends, on the context's clock. */
|
|
105
|
+
this.playhead = 0;
|
|
49
106
|
/** A byte of a sample split across two frames. */
|
|
50
107
|
this.carry = null;
|
|
51
108
|
this.capture = null;
|
|
52
109
|
this.drained = new Set();
|
|
110
|
+
this.stopped = new Set();
|
|
111
|
+
this.scheduled = new Set();
|
|
112
|
+
this.captured = new Set();
|
|
53
113
|
this.interruptions = new Set();
|
|
54
|
-
|
|
114
|
+
this.leadIn = (options.leadInMs ?? 0) / 1000;
|
|
115
|
+
this.latency =
|
|
116
|
+
options.outputLatencyMs === undefined
|
|
117
|
+
? null
|
|
118
|
+
: options.outputLatencyMs / 1000;
|
|
119
|
+
this.output = options.output ?? "speaker";
|
|
120
|
+
session.configure(this.output).catch(() => {
|
|
121
|
+
// swallowed: the library's defaults stand until `prime()` or a press sets the session again.
|
|
122
|
+
});
|
|
55
123
|
this.unwatch = session.watch(({ began, shouldResume }) => {
|
|
124
|
+
if (Date.now() < this.settling)
|
|
125
|
+
return;
|
|
56
126
|
if (began)
|
|
57
127
|
this.interrupt();
|
|
58
128
|
else if (shouldResume && this.interrupted) {
|
|
@@ -67,30 +137,88 @@ export class PhoneAudio {
|
|
|
67
137
|
this.player(this.context?.sampleRate ?? DEFAULT_SAMPLE_RATE);
|
|
68
138
|
}
|
|
69
139
|
play(pcm, frame) {
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
140
|
+
const rate = frame.sample_rate;
|
|
141
|
+
this.player(rate);
|
|
142
|
+
const whole = this.whole(pcm);
|
|
143
|
+
if (whole.byteLength === 0)
|
|
73
144
|
return;
|
|
74
|
-
const
|
|
145
|
+
const context = this.context;
|
|
146
|
+
const samples = pcm16ToFloat(whole);
|
|
147
|
+
const buffer = context.createBuffer(1, samples.length, rate);
|
|
75
148
|
buffer.copyToChannel(samples, 0);
|
|
76
|
-
|
|
149
|
+
// Read once the buffer is made, as near its enqueue as can be: copying
|
|
150
|
+
// a frame over JSI took 11 ms on Android's emulator, and a frame that
|
|
151
|
+
// starts at once was told as starting that long before it was queued.
|
|
152
|
+
const now = context.currentTime;
|
|
153
|
+
if (this.playhead <= now) {
|
|
154
|
+
// The queue ran dry: this frame starts a run, after the lead-in.
|
|
155
|
+
this.playhead = now;
|
|
156
|
+
// Never an empty buffer: the library throws on a length of 0.
|
|
157
|
+
const silence = Math.round(this.leadIn * rate);
|
|
158
|
+
if (silence > 0)
|
|
159
|
+
this.enqueue(context.createBuffer(1, silence, rate));
|
|
160
|
+
}
|
|
161
|
+
const startsAt = this.playhead;
|
|
162
|
+
this.enqueue(buffer);
|
|
163
|
+
this.tell(this.scheduled, {
|
|
164
|
+
pcm: whole,
|
|
165
|
+
sample_rate: rate,
|
|
166
|
+
channels: frame.channels,
|
|
167
|
+
chunk: frame.chunk,
|
|
168
|
+
turn: frame.turn,
|
|
169
|
+
startsAt,
|
|
170
|
+
duration: buffer.duration,
|
|
171
|
+
});
|
|
77
172
|
}
|
|
78
173
|
stop() {
|
|
79
174
|
const was = this.playing.size > 0;
|
|
80
175
|
this.queue?.clearBuffers();
|
|
81
176
|
this.playing.clear();
|
|
177
|
+
this.playhead = 0;
|
|
82
178
|
// Half a sample of audio that will never play now.
|
|
83
179
|
this.carry = null;
|
|
84
|
-
if (was)
|
|
85
|
-
|
|
180
|
+
if (!was)
|
|
181
|
+
return;
|
|
182
|
+
this.tell(this.drained);
|
|
183
|
+
this.tell(this.stopped);
|
|
86
184
|
}
|
|
87
185
|
onDrained(fn) {
|
|
88
|
-
this.drained
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
186
|
+
return add(this.drained, fn);
|
|
187
|
+
}
|
|
188
|
+
now() {
|
|
189
|
+
return this.context?.currentTime ?? 0;
|
|
190
|
+
}
|
|
191
|
+
heardTime() {
|
|
192
|
+
const context = this.context;
|
|
193
|
+
const latency = this.latency ??
|
|
194
|
+
(context?.outputLatency ?? 0) + (context?.baseLatency ?? 0);
|
|
195
|
+
return Math.max(0, this.now() - latency);
|
|
196
|
+
}
|
|
197
|
+
onScheduled(fn) {
|
|
198
|
+
return add(this.scheduled, fn);
|
|
199
|
+
}
|
|
200
|
+
onStopped(fn) {
|
|
201
|
+
return add(this.stopped, fn);
|
|
202
|
+
}
|
|
203
|
+
onCapture(fn) {
|
|
204
|
+
return add(this.captured, fn);
|
|
205
|
+
}
|
|
206
|
+
/** The system's microphone and speaker: none listed, none set. */
|
|
207
|
+
async inputs() {
|
|
208
|
+
return [];
|
|
209
|
+
}
|
|
210
|
+
async outputs() {
|
|
211
|
+
return [];
|
|
92
212
|
}
|
|
93
|
-
|
|
213
|
+
setInput() { }
|
|
214
|
+
setProcessing() { }
|
|
215
|
+
async setOutputDevice() {
|
|
216
|
+
return false;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* At 16 kHz, or at the rate asked when the socket takes it; cancelling
|
|
220
|
+
* echo when hands-free asks and the library can.
|
|
221
|
+
*/
|
|
94
222
|
async startCapture(onPcm, options) {
|
|
95
223
|
// A press is the person going on: the audio is taken back first.
|
|
96
224
|
if (this.interrupted)
|
|
@@ -105,7 +233,22 @@ export class PhoneAudio {
|
|
|
105
233
|
const rate = asked && RECORDING_RATES.includes(asked)
|
|
106
234
|
? asked
|
|
107
235
|
: RECORD_RATE;
|
|
108
|
-
const
|
|
236
|
+
const cancel = options?.echoCancellation === true && session.cancelsEcho();
|
|
237
|
+
try {
|
|
238
|
+
if (cancel)
|
|
239
|
+
await this.enterHandsFree();
|
|
240
|
+
return await this.record(onPcm, rate, cancel);
|
|
241
|
+
}
|
|
242
|
+
catch (failure) {
|
|
243
|
+
await this.leaveHandsFree();
|
|
244
|
+
throw failure;
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
/** The recorder, open: its echo cancelled when `cancel`. */
|
|
248
|
+
async record(onPcm, rate, cancel) {
|
|
249
|
+
const recorder = new AudioRecorder(cancel
|
|
250
|
+
? { iosVoiceProcessing: true, androidInputPreset: "voiceCommunication" }
|
|
251
|
+
: undefined);
|
|
109
252
|
const mine = { recorder, open: true };
|
|
110
253
|
const ready = recorder.onAudioReady({
|
|
111
254
|
sampleRate: rate,
|
|
@@ -118,7 +261,9 @@ export class PhoneAudio {
|
|
|
118
261
|
const at = buffer.sampleRate === rate
|
|
119
262
|
? samples
|
|
120
263
|
: resample(samples, buffer.sampleRate, rate);
|
|
121
|
-
|
|
264
|
+
const pcm = floatToPcm16(at).buffer;
|
|
265
|
+
onPcm(pcm);
|
|
266
|
+
this.tell(this.captured, pcm, rate);
|
|
122
267
|
});
|
|
123
268
|
if (ready.status === "error")
|
|
124
269
|
throw new Error(ready.message);
|
|
@@ -138,8 +283,10 @@ export class PhoneAudio {
|
|
|
138
283
|
async stopCapture() {
|
|
139
284
|
const mine = this.capture;
|
|
140
285
|
this.capture = null;
|
|
141
|
-
if (!mine)
|
|
286
|
+
if (!mine) {
|
|
287
|
+
await this.leaveHandsFree();
|
|
142
288
|
return;
|
|
289
|
+
}
|
|
143
290
|
try {
|
|
144
291
|
// What the recorder hands over until it has stopped is the end of
|
|
145
292
|
// the last word: delivered before this settles.
|
|
@@ -148,6 +295,8 @@ export class PhoneAudio {
|
|
|
148
295
|
finally {
|
|
149
296
|
mine.open = false;
|
|
150
297
|
mine.recorder.clearOnAudioReady();
|
|
298
|
+
// Once the recorder has let go: Android's session is made again.
|
|
299
|
+
await this.leaveHandsFree();
|
|
151
300
|
}
|
|
152
301
|
}
|
|
153
302
|
async resume() {
|
|
@@ -159,41 +308,126 @@ export class PhoneAudio {
|
|
|
159
308
|
this.tell(this.interruptions, false);
|
|
160
309
|
}
|
|
161
310
|
setOutput(output) {
|
|
162
|
-
|
|
311
|
+
this.output = output;
|
|
312
|
+
session.configure(output, this.cancelling).catch(() => {
|
|
313
|
+
// swallowed: the session stays as it was — the route the system chose.
|
|
314
|
+
});
|
|
315
|
+
if (this.cancelling)
|
|
316
|
+
this.reroute();
|
|
317
|
+
}
|
|
318
|
+
/** Hands-free's first step on sound over the reply: lowered, until the words say who it was. */
|
|
319
|
+
duck(on) {
|
|
320
|
+
this.volume = on ? DUCKED : 1;
|
|
321
|
+
if (this.gain)
|
|
322
|
+
this.gain.gain.value = this.volume;
|
|
163
323
|
}
|
|
164
324
|
async handsFree() {
|
|
165
|
-
|
|
325
|
+
if (session.cancelsEcho())
|
|
326
|
+
return { ok: true };
|
|
327
|
+
const version = session.libraryVersion();
|
|
328
|
+
return {
|
|
329
|
+
ok: false,
|
|
330
|
+
reason: version.startsWith("0.") ? PHONE_ECHO : untestedBuild(version),
|
|
331
|
+
};
|
|
166
332
|
}
|
|
167
333
|
onInterruption(fn) {
|
|
168
|
-
this.interruptions
|
|
169
|
-
return () => {
|
|
170
|
-
this.interruptions.delete(fn);
|
|
171
|
-
};
|
|
334
|
+
return add(this.interruptions, fn);
|
|
172
335
|
}
|
|
173
336
|
dispose() {
|
|
337
|
+
this.disposed = true;
|
|
174
338
|
this.stop();
|
|
339
|
+
// Hands-free's session undone with the recorder.
|
|
175
340
|
void this.stopCapture();
|
|
176
341
|
this.unwatch();
|
|
177
342
|
void this.context?.close();
|
|
178
343
|
this.context = null;
|
|
179
344
|
this.queue = null;
|
|
345
|
+
this.gain = null;
|
|
346
|
+
}
|
|
347
|
+
/** Behind what is queued; the playhead moves to its end. */
|
|
348
|
+
enqueue(buffer) {
|
|
349
|
+
this.playing.add(this.queue.enqueueBuffer(buffer));
|
|
350
|
+
this.playhead += buffer.duration;
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* The session hands-free's, Android's reply on the call's path and the
|
|
354
|
+
* call's route: before the recorder starts, so the canceller hears the
|
|
355
|
+
* first reply too.
|
|
356
|
+
*/
|
|
357
|
+
async enterHandsFree() {
|
|
358
|
+
this.cancelling = true;
|
|
359
|
+
this.routed = null;
|
|
360
|
+
await this.reconfigure(true);
|
|
361
|
+
this.unroute ?? (this.unroute = session.watchRoutes(() => this.reroute()));
|
|
362
|
+
this.reroute();
|
|
363
|
+
if (this.context && this.playing.size === 0) {
|
|
364
|
+
this.player(this.context.sampleRate);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
/** Back to push-to-talk's session; the reply leaves the call's path once it has played out. */
|
|
368
|
+
async leaveHandsFree() {
|
|
369
|
+
if (!this.cancelling)
|
|
370
|
+
return;
|
|
371
|
+
this.cancelling = false;
|
|
372
|
+
this.unroute?.();
|
|
373
|
+
this.unroute = null;
|
|
374
|
+
await this.reconfigure(false);
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* The session for hands-free or not — disposed, the mode given back and
|
|
378
|
+
* nothing taken again for a screen gone — its focus's comings and goings
|
|
379
|
+
* no interruption.
|
|
380
|
+
*/
|
|
381
|
+
async reconfigure(handsFree) {
|
|
382
|
+
this.settling = Number.POSITIVE_INFINITY;
|
|
383
|
+
try {
|
|
384
|
+
await session.configure(this.output, handsFree, !this.disposed);
|
|
385
|
+
}
|
|
386
|
+
finally {
|
|
387
|
+
this.settling = Date.now() + SETTLE_MS;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
reroute() {
|
|
391
|
+
session
|
|
392
|
+
.route(this.output, this.routed)
|
|
393
|
+
.then((device) => {
|
|
394
|
+
this.routed = device;
|
|
395
|
+
})
|
|
396
|
+
.catch(() => {
|
|
397
|
+
// swallowed: the system's route stands — the loudspeaker, or the earpiece in a call.
|
|
398
|
+
});
|
|
180
399
|
}
|
|
181
400
|
/** The queue for a reply at `rate`: this one, or a new context's. */
|
|
182
401
|
player(rate) {
|
|
183
|
-
|
|
402
|
+
const profile = this.cancelling && session.android() ? "voiceCommunication" : "media";
|
|
403
|
+
if (this.queue &&
|
|
404
|
+
this.context?.sampleRate === rate &&
|
|
405
|
+
// Onto the call's path or off it, never mid-reply.
|
|
406
|
+
(this.profile === profile || this.playing.size > 0)) {
|
|
184
407
|
return this.queue;
|
|
408
|
+
}
|
|
185
409
|
this.stop();
|
|
186
410
|
void this.context?.close();
|
|
187
|
-
const context = new AudioContext(
|
|
411
|
+
const context = new AudioContext(profile === "media"
|
|
412
|
+
? { sampleRate: rate }
|
|
413
|
+
: { sampleRate: rate, androidOutputProfile: profile });
|
|
414
|
+
this.profile = profile;
|
|
188
415
|
const queue = context.createBufferQueueSource();
|
|
189
|
-
|
|
190
|
-
|
|
416
|
+
const gain = context.createGain();
|
|
417
|
+
gain.gain.value = this.volume;
|
|
418
|
+
queue.connect(gain);
|
|
419
|
+
gain.connect(context.destination);
|
|
420
|
+
const ended = ({ bufferId }) => {
|
|
191
421
|
if (this.queue !== queue)
|
|
192
422
|
return;
|
|
193
423
|
if (this.playing.delete(bufferId) && this.playing.size === 0) {
|
|
194
424
|
this.tell(this.drained);
|
|
195
425
|
}
|
|
196
426
|
};
|
|
427
|
+
// 0.13's name, and 1.0's: each takes the other's as a plain property,
|
|
428
|
+
// set for nothing — and with the wrong one alone no reply ever ends.
|
|
429
|
+
queue.onBufferEnded = ended;
|
|
430
|
+
queue.onbufferended = ended;
|
|
197
431
|
// Never `start()`: 0.13.6's wrapper defaults `offset` to -1 and then
|
|
198
432
|
// throws on its own default (`offset must be a finite non-negative
|
|
199
433
|
// number: -1`), which crashed the first reply on Android's emulator.
|
|
@@ -204,15 +438,16 @@ export class PhoneAudio {
|
|
|
204
438
|
this.quiet(context);
|
|
205
439
|
this.context = context;
|
|
206
440
|
this.queue = queue;
|
|
441
|
+
this.gain = gain;
|
|
207
442
|
return queue;
|
|
208
443
|
}
|
|
209
444
|
/**
|
|
210
|
-
* The wire's PCM
|
|
211
|
-
* the edge sends whole samples (`apps/edge/src/voice/tts.ts`),
|
|
212
|
-
* byte dropped shifts every byte after it and turns the rest
|
|
213
|
-
* the bug that once made the console's playback white noise.
|
|
445
|
+
* The wire's PCM in whole samples. **A split sample is carried, never
|
|
446
|
+
* dropped**: the edge sends whole samples (`apps/edge/src/voice/tts.ts`),
|
|
447
|
+
* but an odd byte dropped shifts every byte after it and turns the rest
|
|
448
|
+
* into noise — the bug that once made the console's playback white noise.
|
|
214
449
|
*/
|
|
215
|
-
|
|
450
|
+
whole(pcm) {
|
|
216
451
|
let bytes = new Uint8Array(pcm);
|
|
217
452
|
if (this.carry !== null) {
|
|
218
453
|
const joined = new Uint8Array(bytes.length + 1);
|
|
@@ -225,7 +460,7 @@ export class PhoneAudio {
|
|
|
225
460
|
this.carry = bytes[bytes.length - 1];
|
|
226
461
|
bytes = bytes.slice(0, -1);
|
|
227
462
|
}
|
|
228
|
-
return
|
|
463
|
+
return bytes.buffer;
|
|
229
464
|
}
|
|
230
465
|
/** The audio was taken: the reply waits, the recording is thrown away. */
|
|
231
466
|
interrupt() {
|
|
@@ -252,11 +487,30 @@ export class PhoneAudio {
|
|
|
252
487
|
// swallowed: an interruption may have stopped the engine already.
|
|
253
488
|
});
|
|
254
489
|
}
|
|
490
|
+
/**
|
|
491
|
+
* Call each of the app's handlers. One that throws is rethrown on its own
|
|
492
|
+
* turn, where the app's error reporting sees it — not here, where it
|
|
493
|
+
* would stop the player or the recorder halfway.
|
|
494
|
+
*/
|
|
255
495
|
tell(listeners, ...args) {
|
|
256
|
-
for (const fn of listeners)
|
|
257
|
-
|
|
496
|
+
for (const fn of listeners) {
|
|
497
|
+
try {
|
|
498
|
+
fn(...args);
|
|
499
|
+
}
|
|
500
|
+
catch (error) {
|
|
501
|
+
setTimeout(() => {
|
|
502
|
+
throw error;
|
|
503
|
+
});
|
|
504
|
+
}
|
|
505
|
+
}
|
|
258
506
|
}
|
|
259
507
|
}
|
|
508
|
+
function add(listeners, fn) {
|
|
509
|
+
listeners.add(fn);
|
|
510
|
+
return () => {
|
|
511
|
+
listeners.delete(fn);
|
|
512
|
+
};
|
|
513
|
+
}
|
|
260
514
|
export const createAudio = (options) => new PhoneAudio(options);
|
|
261
515
|
/** Ask for the microphone, if it was not granted: whether it is now. */
|
|
262
516
|
export const requestMicrophone = () => session.microphone();
|
package/dist/audio.web.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { CreateAudio } from "./device-audio.js";
|
|
2
|
+
/** The lead-in, the microphone and its processing are the browser's; `output` and `outputLatencyMs` the phone's. */
|
|
2
3
|
export declare const createAudio: CreateAudio;
|
|
3
4
|
/** Ask for the microphone, if it was not granted: whether it is now. */
|
|
4
5
|
export declare function requestMicrophone(): Promise<boolean>;
|
package/dist/audio.web.js
CHANGED
|
@@ -1,45 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A browser's player and recorder: what Metro picks for `./audio` on the
|
|
3
|
+
* web (`index.ts`), and what anything picking no platform file gets
|
|
4
|
+
* (`audio.ts`). `react-native-audio-api` has no recorder, no queue and no
|
|
5
|
+
* real permission request on the web, so this is the client's own
|
|
6
|
+
* `@lumavox/client/web-audio` — the console's — with the phone's extras
|
|
7
|
+
* answered as a browser can.
|
|
8
|
+
*
|
|
9
|
+
* - **`prime()` must run in the press that starts the conversation**:
|
|
10
|
+
* browsers start sound only from a gesture.
|
|
11
|
+
* - **No interruptions are told**: a browser pauses a tab's audio itself.
|
|
12
|
+
* - **No earpiece**: `setOutput` does nothing; the speaker is chosen by its
|
|
13
|
+
* id (`setOutputDevice`), where the browser lets a page.
|
|
14
|
+
* - **Hands-free where the browser cancels echo**, as the client's says
|
|
15
|
+
* (`handsFree`), its reply lowered while the person may be talking.
|
|
16
|
+
* - **The client's adapter itself, with the phone's extras added**: every
|
|
17
|
+
* method it has — the clock, the frames, the devices — is passed through
|
|
18
|
+
* by being it, so none can be left out.
|
|
19
|
+
* - **The import needs Metro's package `exports`** — on by default from
|
|
20
|
+
* React Native 0.79 and Expo SDK 53; before, an app sets
|
|
21
|
+
* `unstable_enablePackageExports` (the docs' React Native page).
|
|
22
|
+
*/
|
|
1
23
|
import { createWebAudio } from "@lumavox/client/web-audio";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
}
|
|
13
|
-
stop() {
|
|
14
|
-
this.web.stop();
|
|
15
|
-
}
|
|
16
|
-
onDrained(fn) {
|
|
17
|
-
return this.web.onDrained(fn);
|
|
18
|
-
}
|
|
19
|
-
startCapture(onPcm, options) {
|
|
20
|
-
return this.web.startCapture(onPcm, options);
|
|
21
|
-
}
|
|
22
|
-
stopCapture() {
|
|
23
|
-
return this.web.stopCapture();
|
|
24
|
-
}
|
|
25
|
-
dispose() {
|
|
26
|
-
this.web.dispose();
|
|
27
|
-
}
|
|
28
|
-
resume() {
|
|
29
|
-
return this.web.prime();
|
|
30
|
-
}
|
|
31
|
-
setOutput(_output) { }
|
|
32
|
-
handsFree() {
|
|
33
|
-
return this.web.handsFree();
|
|
34
|
-
}
|
|
35
|
-
duck(on) {
|
|
36
|
-
this.web.duck(on);
|
|
37
|
-
}
|
|
38
|
-
onInterruption(_fn) {
|
|
39
|
-
return () => { };
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
export const createAudio = () => new BrowserAudio();
|
|
24
|
+
/** The lead-in, the microphone and its processing are the browser's; `output` and `outputLatencyMs` the phone's. */
|
|
25
|
+
export const createAudio = (options) => {
|
|
26
|
+
const web = createWebAudio(options);
|
|
27
|
+
return Object.assign(web, {
|
|
28
|
+
interrupted: false,
|
|
29
|
+
resume: () => web.prime(),
|
|
30
|
+
setOutput: () => { },
|
|
31
|
+
onInterruption: () => () => { },
|
|
32
|
+
});
|
|
33
|
+
};
|
|
43
34
|
/** Ask for the microphone, if it was not granted: whether it is now. */
|
|
44
35
|
export async function requestMicrophone() {
|
|
45
36
|
try {
|
package/dist/device-audio.d.ts
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* What `createAudio()` answers on every platform — a phone's
|
|
3
3
|
* (`audio.native.ts`) and a browser's (`audio.web.ts`) — so an app written
|
|
4
|
-
* once runs on both: the `AgentAudio` that `useAgent` drives,
|
|
4
|
+
* once runs on both: the `AgentAudio` that `useAgent` drives, the clock
|
|
5
|
+
* and the frames an app draws against (`PlaybackClock`), the devices a
|
|
6
|
+
* browser lets it choose (`DeviceChoices`, none on a phone), and what an
|
|
5
7
|
* app does with the device itself.
|
|
6
8
|
*/
|
|
7
|
-
import type { AgentAudio } from "@lumavox/client";
|
|
9
|
+
import type { AgentAudio, DeviceChoices, PlaybackClock, Processing } from "@lumavox/client";
|
|
8
10
|
/** Where the agent is heard with no headphones on. */
|
|
9
11
|
export type Output = "speaker" | "earpiece";
|
|
10
12
|
export interface AudioOptions {
|
|
@@ -15,8 +17,26 @@ export interface AudioOptions {
|
|
|
15
17
|
* 1.0 lets the earpiece be chosen.
|
|
16
18
|
*/
|
|
17
19
|
output?: Output;
|
|
20
|
+
/**
|
|
21
|
+
* Milliseconds a reply's first frame — or one after a gap — waits before
|
|
22
|
+
* it plays, for an app that reads each frame first (`onScheduled`): as
|
|
23
|
+
* much is added to the first word heard. None on a phone unless set, 60
|
|
24
|
+
* in a browser.
|
|
25
|
+
*/
|
|
26
|
+
leadInMs?: number;
|
|
27
|
+
/**
|
|
28
|
+
* Milliseconds from the phone's player to the ear, which `heardTime()`
|
|
29
|
+
* takes off the clock — and half-duplex waits out: `react-native-audio-api`
|
|
30
|
+
* 0.13 cannot say it, so `0` unless set — Bluetooth earbuds add about 150
|
|
31
|
+
* to 250. A browser says its own.
|
|
32
|
+
*/
|
|
33
|
+
outputLatencyMs?: number;
|
|
34
|
+
/** The microphone, by its `inputs()` id, in a browser; a phone's system chooses. */
|
|
35
|
+
input?: string;
|
|
36
|
+
/** The browser's cleanup of the microphone; a phone's is the system's. */
|
|
37
|
+
processing?: Processing;
|
|
18
38
|
}
|
|
19
|
-
export interface DeviceAudio extends AgentAudio {
|
|
39
|
+
export interface DeviceAudio extends AgentAudio, PlaybackClock, DeviceChoices {
|
|
20
40
|
/**
|
|
21
41
|
* Get ready to play. **On the web, call it from the press that starts
|
|
22
42
|
* the conversation**: browsers start sound only from a gesture. On a
|
|
@@ -37,7 +57,9 @@ export interface DeviceAudio extends AgentAudio {
|
|
|
37
57
|
/**
|
|
38
58
|
* Whether hands-free may run (`useAgent`'s `listen`): on the web where
|
|
39
59
|
* the browser cancels echo; on a phone not until `react-native-audio-api`
|
|
40
|
-
* 1.0 does — with headphones on, an app passes `assumeNoEcho`.
|
|
60
|
+
* 1.0 does — with headphones on, an app passes `assumeNoEcho`. Half-duplex
|
|
61
|
+
* (`listen({ halfDuplex: true })`) needs no echo cancelled, and runs
|
|
62
|
+
* everywhere.
|
|
41
63
|
*/
|
|
42
64
|
handsFree(): Promise<{
|
|
43
65
|
ok: true;
|
|
@@ -45,6 +67,8 @@ export interface DeviceAudio extends AgentAudio {
|
|
|
45
67
|
ok: false;
|
|
46
68
|
reason: string;
|
|
47
69
|
}>;
|
|
70
|
+
/** Hands-free's first step: the reply quieter while it may be talked over, or as before. */
|
|
71
|
+
duck(on: boolean): void;
|
|
48
72
|
}
|
|
49
73
|
/** `createAudio`'s signature, which each platform's file implements. */
|
|
50
74
|
export type CreateAudio = (options?: AudioOptions) => DeviceAudio;
|
package/dist/session.d.ts
CHANGED
|
@@ -1,6 +1,44 @@
|
|
|
1
|
+
import { type CommunicationDevice } from "react-native-audio-api";
|
|
1
2
|
import type { Output } from "./device-audio.js";
|
|
2
|
-
/**
|
|
3
|
-
|
|
3
|
+
/**
|
|
4
|
+
* **The one `react-native-audio-api` build hands-free runs on before 1.0**
|
|
5
|
+
* (D17, D18): 1.0 is the first with echo cancellation — iOS's voice
|
|
6
|
+
* processing, Android's voice-communication audio — and is not released
|
|
7
|
+
* (no date, 2026-11-02), so one nightly, read in its source and tested,
|
|
8
|
+
* never "the latest": nightlies change under an app, and its iOS voice
|
|
9
|
+
* processing was weeks old. This one is commit 6fd4c68, built unchanged on
|
|
10
|
+
* 2026-10-31, 11-01 and 11-02 — the build npm's `audio-api-nightly` tag
|
|
11
|
+
* named on 2026-11-02. **Moved only for a fix we need, after a test on a
|
|
12
|
+
* real iPhone and Android; 1.0 replaces it.** Every other build, 0.13
|
|
13
|
+
* included, refuses hands-free and runs push-to-talk and half-duplex.
|
|
14
|
+
* `package.json`'s peer range names it too (`test/package.test.ts`).
|
|
15
|
+
*/
|
|
16
|
+
export declare const PINNED_NIGHTLY = "1.0.0-nightly-6fd4c68-20261102";
|
|
17
|
+
/** The library's version, as installed. */
|
|
18
|
+
export declare const libraryVersion: () => string;
|
|
19
|
+
/** Whether this is the build hands-free runs on: echo can be cancelled. */
|
|
20
|
+
export declare const cancelsEcho: () => boolean;
|
|
21
|
+
/** Android, where the reply must be on the call's path for its echo to be cancelled. */
|
|
22
|
+
export declare const android: () => boolean;
|
|
23
|
+
/**
|
|
24
|
+
* The audio session's category, mode and options for `output`, for
|
|
25
|
+
* hands-free or not. Settles at once but for Android's session on 1.0,
|
|
26
|
+
* which takes a mode only as it is made active, so is made again.
|
|
27
|
+
*/
|
|
28
|
+
export declare function configure(output: Output, handsFree?: boolean,
|
|
29
|
+
/** `false` when the adapter is gone: the mode given back, the session left inactive. */
|
|
30
|
+
active?: boolean): Promise<void>;
|
|
31
|
+
/**
|
|
32
|
+
* Hands-free on Android: the call's voice to Bluetooth earbuds when they
|
|
33
|
+
* are connected — and their microphone with it (D8) — else to `output`.
|
|
34
|
+
* Asked again when a device comes or goes, never twice the same: the
|
|
35
|
+
* library's own Bluetooth link sets the device too, and its change is told
|
|
36
|
+
* as a route's. `asked`, the last asked; answers what is now. Nothing on
|
|
37
|
+
* iOS, whose route follows `allowBluetoothHFP` by itself, nor on 0.13.
|
|
38
|
+
*/
|
|
39
|
+
export declare function route(output: Output, asked: CommunicationDevice | null): Promise<CommunicationDevice | null>;
|
|
40
|
+
/** Every change of the devices — earbuds connected, taken out — until the answer is called. */
|
|
41
|
+
export declare function watchRoutes(fn: () => void): () => void;
|
|
4
42
|
/** Make the session active: before the first sound, and after an interruption. */
|
|
5
43
|
export declare function activate(): Promise<void>;
|
|
6
44
|
/** Whether the app may record: asked only when not granted yet. */
|
package/dist/session.js
CHANGED
|
@@ -1,24 +1,33 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The phone's audio session: how the app's audio sits beside the system's
|
|
3
3
|
* — the speaker, the earbuds, calls and other apps. With `audio.native.ts`
|
|
4
|
-
* the only file that touches `react-native-audio-api`,
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* the only file that touches `react-native-audio-api`, and **the one that
|
|
5
|
+
* knows which of the library it has**: 0.13, or the one 1.0 nightly
|
|
6
|
+
* hands-free runs on (`PINNED_NIGHTLY`), whose session API is renamed
|
|
7
|
+
* (`setSystemOptions`) — told apart by what the library has, at run time,
|
|
8
|
+
* so one build of this package runs on both.
|
|
7
9
|
*
|
|
8
10
|
* - **Play and record at once, from the loudspeaker** (`playAndRecord`,
|
|
9
11
|
* `defaultToSpeaker`): without it iOS plays a recording app's audio from
|
|
10
12
|
* the earpiece. Headphones and Bluetooth take over when connected; the
|
|
11
13
|
* earpiece is the app's choice (`Output`). Android keeps the loudspeaker
|
|
12
|
-
*
|
|
13
|
-
* earpiece.
|
|
14
|
+
* but in hands-free: the library sets no communication mode otherwise,
|
|
15
|
+
* and so chooses no earpiece.
|
|
14
16
|
* - **The earbuds' microphone on iPhones** (`allowBluetoothHFP`): their
|
|
15
17
|
* hands-free profile, at call quality, for the whole conversation — not
|
|
16
|
-
* switched per press, since every switch moves the route and
|
|
17
|
-
* engine can die on a Bluetooth route change (its #1209
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
18
|
+
* switched per press, since every switch moves the route and the
|
|
19
|
+
* library's engine can die on a Bluetooth route change (its #1209).
|
|
20
|
+
* - **`default` mode, not `voiceChat`**, for push-to-talk and half-duplex:
|
|
21
|
+
* neither needs echo cancelled — the reply stops while the person speaks,
|
|
22
|
+
* or nothing is heard while it plays — and `voiceChat` plays at the call
|
|
23
|
+
* volume, quieter on the loudspeaker.
|
|
24
|
+
* - **Hands-free, on the pinned nightly** (`handsFree`): iOS's
|
|
25
|
+
* `videoChat` — set before the recorder's voice processing, which would
|
|
26
|
+
* otherwise move the session to `voiceChat` and the earpiece — and
|
|
27
|
+
* Android's communication mode, its voice to the loudspeaker (or the
|
|
28
|
+
* earpiece asked), to Bluetooth earbuds when they are connected: the
|
|
29
|
+
* canceller hears only audio on the call's path, and the earbuds'
|
|
30
|
+
* microphone comes with it (D8).
|
|
22
31
|
* - **Interruptions are watched** (`observeAudioInterruptions`): a call,
|
|
23
32
|
* Siri or another app on iOS, a loss of audio focus on Android — which
|
|
24
33
|
* watching *asks for*, so other apps' audio stops while a conversation is
|
|
@@ -26,17 +35,104 @@
|
|
|
26
35
|
* audio in this release (the plugin turns the library's off), so the
|
|
27
36
|
* conversation pauses with the screen and goes on when it is back.
|
|
28
37
|
*/
|
|
29
|
-
import { AppState } from "react-native";
|
|
38
|
+
import { AppState, Platform } from "react-native";
|
|
30
39
|
import { AudioManager } from "react-native-audio-api";
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
40
|
+
import library from "react-native-audio-api/package.json";
|
|
41
|
+
/**
|
|
42
|
+
* **The one `react-native-audio-api` build hands-free runs on before 1.0**
|
|
43
|
+
* (D17, D18): 1.0 is the first with echo cancellation — iOS's voice
|
|
44
|
+
* processing, Android's voice-communication audio — and is not released
|
|
45
|
+
* (no date, 2026-11-02), so one nightly, read in its source and tested,
|
|
46
|
+
* never "the latest": nightlies change under an app, and its iOS voice
|
|
47
|
+
* processing was weeks old. This one is commit 6fd4c68, built unchanged on
|
|
48
|
+
* 2026-10-31, 11-01 and 11-02 — the build npm's `audio-api-nightly` tag
|
|
49
|
+
* named on 2026-11-02. **Moved only for a fix we need, after a test on a
|
|
50
|
+
* real iPhone and Android; 1.0 replaces it.** Every other build, 0.13
|
|
51
|
+
* included, refuses hands-free and runs push-to-talk and half-duplex.
|
|
52
|
+
* `package.json`'s peer range names it too (`test/package.test.ts`).
|
|
53
|
+
*/
|
|
54
|
+
export const PINNED_NIGHTLY = "1.0.0-nightly-6fd4c68-20261102";
|
|
55
|
+
/** The library's version, as installed. */
|
|
56
|
+
export const libraryVersion = () => library.version;
|
|
57
|
+
/** 1.0's session API: `setSystemOptions` for 0.13's `setAudioSessionOptions`. */
|
|
58
|
+
const isOne = () => typeof AudioManager.setSystemOptions === "function";
|
|
59
|
+
/** Whether this is the build hands-free runs on: echo can be cancelled. */
|
|
60
|
+
export const cancelsEcho = () => isOne() && library.version === PINNED_NIGHTLY;
|
|
61
|
+
/** Android, where the reply must be on the call's path for its echo to be cancelled. */
|
|
62
|
+
export const android = () => Platform.OS === "android";
|
|
63
|
+
/**
|
|
64
|
+
* The audio session's category, mode and options for `output`, for
|
|
65
|
+
* hands-free or not. Settles at once but for Android's session on 1.0,
|
|
66
|
+
* which takes a mode only as it is made active, so is made again.
|
|
67
|
+
*/
|
|
68
|
+
export async function configure(output, handsFree = false,
|
|
69
|
+
/** `false` when the adapter is gone: the mode given back, the session left inactive. */
|
|
70
|
+
active = true) {
|
|
71
|
+
const options = output === "speaker"
|
|
72
|
+
? ["defaultToSpeaker", "allowBluetoothHFP"]
|
|
73
|
+
: ["allowBluetoothHFP"];
|
|
74
|
+
// Voice processing takes a mode that is no call's to `voiceChat` itself,
|
|
75
|
+
// and the library then sets its own again on every activation: a chat
|
|
76
|
+
// mode set first — `videoChat`, the loudspeaker's, or `voiceChat`, the
|
|
77
|
+
// earpiece's. Never without it: the call volume.
|
|
78
|
+
const mode = !handsFree
|
|
79
|
+
? "default"
|
|
80
|
+
: output === "speaker"
|
|
81
|
+
? "videoChat"
|
|
82
|
+
: "voiceChat";
|
|
83
|
+
if (!isOne()) {
|
|
84
|
+
AudioManager.setAudioSessionOptions?.({
|
|
85
|
+
iosCategory: "playAndRecord",
|
|
86
|
+
iosMode: mode,
|
|
87
|
+
iosOptions: options,
|
|
88
|
+
});
|
|
89
|
+
return;
|
|
90
|
+
}
|
|
91
|
+
// Both groups every time: one left out is set to its defaults.
|
|
92
|
+
AudioManager.setSystemOptions?.({
|
|
93
|
+
ios: { category: "playAndRecord", mode, options },
|
|
94
|
+
android: handsFree
|
|
95
|
+
? { mode: "inCommunication", communicationDevice: output }
|
|
96
|
+
: {},
|
|
39
97
|
});
|
|
98
|
+
if (android() && communicating !== handsFree) {
|
|
99
|
+
// Its communication mode is set, and the mode before it given back,
|
|
100
|
+
// only as the session is made active and inactive — which gives up the
|
|
101
|
+
// audio focus and takes it again: told as an interruption, which the
|
|
102
|
+
// adapter takes for none (`audio.native.ts`, `settling`).
|
|
103
|
+
communicating = handsFree;
|
|
104
|
+
await AudioManager.setAudioSessionActivity(false);
|
|
105
|
+
if (active)
|
|
106
|
+
await AudioManager.setAudioSessionActivity(true);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
/** Android's communication mode is on: hands-free's session was made. */
|
|
110
|
+
let communicating = false;
|
|
111
|
+
/**
|
|
112
|
+
* Hands-free on Android: the call's voice to Bluetooth earbuds when they
|
|
113
|
+
* are connected — and their microphone with it (D8) — else to `output`.
|
|
114
|
+
* Asked again when a device comes or goes, never twice the same: the
|
|
115
|
+
* library's own Bluetooth link sets the device too, and its change is told
|
|
116
|
+
* as a route's. `asked`, the last asked; answers what is now. Nothing on
|
|
117
|
+
* iOS, whose route follows `allowBluetoothHFP` by itself, nor on 0.13.
|
|
118
|
+
*/
|
|
119
|
+
export async function route(output, asked) {
|
|
120
|
+
if (!android() || !isOne() || !AudioManager.getDevicesInfo)
|
|
121
|
+
return asked;
|
|
122
|
+
const { availableOutputs } = await AudioManager.getDevicesInfo();
|
|
123
|
+
// Earbuds with a microphone, as the library names Android's device
|
|
124
|
+
// types: a speaker on Bluetooth alone (A2DP) has none, and with
|
|
125
|
+
// `systemDefault` the call would go to the earpiece.
|
|
126
|
+
const earbuds = availableOutputs.some((device) => /^Bluetooth (SCO|LE Headset)$/.test(device.category));
|
|
127
|
+
const device = earbuds ? "systemDefault" : output;
|
|
128
|
+
if (device !== asked)
|
|
129
|
+
await AudioManager.setCommunicationDevice?.(device);
|
|
130
|
+
return device;
|
|
131
|
+
}
|
|
132
|
+
/** Every change of the devices — earbuds connected, taken out — until the answer is called. */
|
|
133
|
+
export function watchRoutes(fn) {
|
|
134
|
+
const subscription = AudioManager.addSystemEventListener("routeChange", () => fn());
|
|
135
|
+
return () => subscription.remove();
|
|
40
136
|
}
|
|
41
137
|
/** Make the session active: before the first sound, and after an interruption. */
|
|
42
138
|
export function activate() {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lumavox/react-native",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Voice conversations with Lumavox agents in a React Native app, on iOS, Android and the web: the client, the React hooks, and push-to-talk audio on react-native-audio-api.",
|
|
3
|
+
"version": "0.17.0",
|
|
4
|
+
"description": "Voice conversations with Lumavox agents in a React Native app, on iOS, Android and the web: the client, the React hooks, and push-to-talk, half-duplex and hands-free audio on react-native-audio-api.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"files": [
|
|
7
7
|
"dist",
|
|
@@ -30,13 +30,13 @@
|
|
|
30
30
|
"typecheck": "tsc --noEmit && tsc --noEmit -p test"
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@lumavox/client": "0.
|
|
34
|
-
"@lumavox/react": "0.
|
|
33
|
+
"@lumavox/client": "0.17.0",
|
|
34
|
+
"@lumavox/react": "0.17.0"
|
|
35
35
|
},
|
|
36
36
|
"peerDependencies": {
|
|
37
37
|
"react": ">=18",
|
|
38
38
|
"react-native": ">=0.76",
|
|
39
|
-
"react-native-audio-api": ">=0.13.6 <0.14"
|
|
39
|
+
"react-native-audio-api": ">=0.13.6 <0.14 || 1.0.0-nightly-6fd4c68-20261102"
|
|
40
40
|
},
|
|
41
41
|
"peerDependenciesMeta": {
|
|
42
42
|
"react-native": {
|