@lumavox/react-native 0.0.0-stage → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +61 -2
- package/app.plugin.js +98 -0
- package/dist/audio.d.ts +8 -0
- package/dist/audio.js +8 -0
- package/dist/audio.native.d.ts +50 -0
- package/dist/audio.native.js +250 -0
- package/dist/audio.web.d.ts +4 -0
- package/dist/audio.web.js +49 -0
- package/dist/device-audio.d.ts +39 -0
- package/dist/device-audio.js +1 -0
- package/dist/index.d.ts +17 -0
- package/dist/index.js +19 -0
- package/dist/session.d.ts +15 -0
- package/dist/session.js +80 -0
- package/package.json +56 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Lumavox
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,62 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @lumavox/react-native
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Voice conversations with Lumavox agents in a React Native app, on iOS,
|
|
4
|
+
Android and the web — **one install**: the client (`@lumavox/client`), the
|
|
5
|
+
hooks (`@lumavox/react`), and the audio they lack, a player and a recorder
|
|
6
|
+
for **push-to-talk**, on
|
|
7
|
+
[`react-native-audio-api`](https://github.com/software-mansion/react-native-audio-api)
|
|
8
|
+
on a phone and Web Audio in a browser.
|
|
9
|
+
|
|
10
|
+
```sh
|
|
11
|
+
npm install @lumavox/react-native react-native-audio-api
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
React Native 0.76 or later on the New Architecture, iOS 17 and Android 12
|
|
15
|
+
or later; with Expo, a development build — never Expo Go. In `app.json`:
|
|
16
|
+
|
|
17
|
+
```json
|
|
18
|
+
{ "expo": { "plugins": [["@lumavox/react-native", { "microphonePermission": "Hold to talk to your coach." }]] } }
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
**Never list `react-native-audio-api`'s plugin as well**: this one runs it
|
|
22
|
+
with background audio off and every permission listed, and it runs once.
|
|
23
|
+
|
|
24
|
+
```tsx
|
|
25
|
+
import { AgentProvider, createAudio, createClient, useAgent } from "@lumavox/react-native";
|
|
26
|
+
|
|
27
|
+
const lumavox = createClient({
|
|
28
|
+
// Your server mints the key with your API key (POST /v1/ephemeral-keys).
|
|
29
|
+
getToken: async () => (await fetch(keyServer, { method: "POST" })).text(),
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
function Talk({ agentId }: { agentId: string }) {
|
|
33
|
+
const audio = useMemo(() => createAudio(), []);
|
|
34
|
+
useEffect(() => () => audio.dispose(), [audio]);
|
|
35
|
+
const agent = useAgent({ agentId }, { audio });
|
|
36
|
+
return (
|
|
37
|
+
<Pressable
|
|
38
|
+
onPressIn={() => {
|
|
39
|
+
void audio.prime(); // in the press: the web needs it
|
|
40
|
+
void agent.startRecording();
|
|
41
|
+
}}
|
|
42
|
+
onPressOut={() => void agent.stopRecording()}
|
|
43
|
+
>
|
|
44
|
+
<Text>{agent.recording ? "Listening…" : "Hold to talk"}</Text>
|
|
45
|
+
</Pressable>
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
// <AgentProvider client={lumavox}><Talk agentId="agt_..." /></AgentProvider>
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
- Phones record at 16 kHz, browsers at 24 kHz; the agent plays from the
|
|
52
|
+
loudspeaker (`createAudio({ output: "earpiece" })` on iPhones), and
|
|
53
|
+
Bluetooth earbuds' microphone is used on iPhones.
|
|
54
|
+
- A call, Siri or another app pauses the reply and throws away a recording
|
|
55
|
+
in progress — `interrupted`, `onAudioInterrupted`, `onAudioResumed`; so
|
|
56
|
+
does leaving the screen: no background audio in this release.
|
|
57
|
+
- Hands-free and background audio come with `react-native-audio-api` 1.0.
|
|
58
|
+
|
|
59
|
+
The guide, with the bare React Native setup: the docs' **SDKs** tab,
|
|
60
|
+
*React Native*.
|
|
61
|
+
|
|
62
|
+
MIT, copyright Lumavox.
|
package/app.plugin.js
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The Expo config plugin: `"plugins": [["@lumavox/react-native", { … }]]`
|
|
3
|
+
* in `app.json`. It runs `react-native-audio-api`'s own plugin with every
|
|
4
|
+
* option set here, because that plugin's defaults are wrong for a
|
|
5
|
+
* conversation and it cannot be set twice:
|
|
6
|
+
*
|
|
7
|
+
* - **Background is on by default there** — iOS's `audio` background mode
|
|
8
|
+
* and an Android `mediaPlayback` foreground service. A conversation
|
|
9
|
+
* pauses with the screen in this release (background is the app's choice
|
|
10
|
+
* from hands-free on), so both are turned off.
|
|
11
|
+
* - **`androidPermissions` replaces its list**, which lacks
|
|
12
|
+
* `RECORD_AUDIO`; nothing removes a permission. Ours is listed in full:
|
|
13
|
+
* `RECORD_AUDIO`, and `MODIFY_AUDIO_SETTINGS` for the communication mode
|
|
14
|
+
* the library's 1.0 sets. An app's own are added to it.
|
|
15
|
+
* - **It runs once** (`createRunOncePlugin`): listed in `app.json` as well,
|
|
16
|
+
* the second listing's options are dropped without a word — so, listed
|
|
17
|
+
* before this one, it is refused here, and the docs say never to list it.
|
|
18
|
+
*
|
|
19
|
+
* CommonJS, as Expo loads it with `require`: the package has no
|
|
20
|
+
* `"type": "module"` for that reason.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** The library's plugin's name in Expo's record of the plugins run. */
|
|
24
|
+
const LIBRARY = "react-native-audio-api";
|
|
25
|
+
|
|
26
|
+
const PERMISSIONS = [
|
|
27
|
+
"android.permission.RECORD_AUDIO",
|
|
28
|
+
"android.permission.MODIFY_AUDIO_SETTINGS",
|
|
29
|
+
];
|
|
30
|
+
|
|
31
|
+
/** iOS shows it when the app first asks for the microphone; the app's own words are better. */
|
|
32
|
+
const MICROPHONE = "Hold to talk to the assistant.";
|
|
33
|
+
|
|
34
|
+
/** What this plugin takes; anything else is a mistake, refused. */
|
|
35
|
+
const OPTIONS = [
|
|
36
|
+
"microphonePermission",
|
|
37
|
+
"androidPermissions",
|
|
38
|
+
"disableFFmpeg",
|
|
39
|
+
"disableStaticExternalLibs",
|
|
40
|
+
];
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The library's plugin's options for ours.
|
|
44
|
+
*
|
|
45
|
+
* @param {{
|
|
46
|
+
* microphonePermission?: string,
|
|
47
|
+
* androidPermissions?: string[],
|
|
48
|
+
* disableFFmpeg?: boolean,
|
|
49
|
+
* disableStaticExternalLibs?: boolean,
|
|
50
|
+
* }} [options]
|
|
51
|
+
*/
|
|
52
|
+
function audioApiOptions(options = {}) {
|
|
53
|
+
for (const name of Object.keys(options)) {
|
|
54
|
+
if (!OPTIONS.includes(name)) {
|
|
55
|
+
throw new Error(
|
|
56
|
+
`@lumavox/react-native's plugin takes ${OPTIONS.join(", ")}, not "${name}".`,
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return {
|
|
61
|
+
iosMicrophonePermission: options.microphonePermission ?? MICROPHONE,
|
|
62
|
+
iosBackgroundMode: false,
|
|
63
|
+
androidPermissions: [
|
|
64
|
+
...new Set([...PERMISSIONS, ...(options.androidPermissions ?? [])]),
|
|
65
|
+
],
|
|
66
|
+
androidForegroundService: false,
|
|
67
|
+
androidFSTypes: [],
|
|
68
|
+
androidFSStopWithTask: true,
|
|
69
|
+
// The library's own defaults: FFmpeg decodes compressed files, which a
|
|
70
|
+
// conversation's raw PCM never needs — `true` makes the app smaller.
|
|
71
|
+
disableFFmpeg: options.disableFFmpeg ?? false,
|
|
72
|
+
disableStaticExternalLibs: options.disableStaticExternalLibs ?? false,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* @param {any} config Expo's app config.
|
|
78
|
+
* @param {Parameters<typeof audioApiOptions>[0]} [options]
|
|
79
|
+
* @param {(config: any, options: object) => any} [library] the library's
|
|
80
|
+
* plugin; Expo passes none, the tests a stand-in.
|
|
81
|
+
*/
|
|
82
|
+
function withLumavox(config, options, library) {
|
|
83
|
+
if (config?._internal?.pluginHistory?.[LIBRARY]) {
|
|
84
|
+
throw new Error(
|
|
85
|
+
`${LIBRARY}'s plugin is listed in app.json beside @lumavox/react-native's: remove it — @lumavox/react-native passes it every option, and it runs only once.`,
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
const run = library ?? loadLibrary();
|
|
89
|
+
return run(config, audioApiOptions(options));
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function loadLibrary() {
|
|
93
|
+
const plugin = require(`${LIBRARY}/app.plugin`);
|
|
94
|
+
return plugin.default ?? plugin;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
module.exports = withLumavox;
|
|
98
|
+
module.exports.audioApiOptions = audioApiOptions;
|
package/dist/audio.d.ts
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `createAudio` and `requestMicrophone` for whatever picks no platform
|
|
3
|
+
* file — TypeScript, a test runner, a bundler that is not React Native's:
|
|
4
|
+
* the browser's. Metro, React Native's bundler, takes `audio.native.ts` on
|
|
5
|
+
* iOS and Android and `audio.web.ts` on the web, since `index.ts` imports
|
|
6
|
+
* `./audio` with no extension; both answer `CreateAudio`.
|
|
7
|
+
*/
|
|
8
|
+
export { createAudio, requestMicrophone } from "./audio.web.js";
|
package/dist/audio.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `createAudio` and `requestMicrophone` for whatever picks no platform
|
|
3
|
+
* file — TypeScript, a test runner, a bundler that is not React Native's:
|
|
4
|
+
* the browser's. Metro, React Native's bundler, takes `audio.native.ts` on
|
|
5
|
+
* iOS and Android and `audio.web.ts` on the web, since `index.ts` imports
|
|
6
|
+
* `./audio` with no extension; both answer `CreateAudio`.
|
|
7
|
+
*/
|
|
8
|
+
export { createAudio, requestMicrophone } from "./audio.web.js";
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { AudioOptions, CreateAudio, DeviceAudio, Output } from "./device-audio.js";
|
|
2
|
+
/** Hz the microphone records at unless a rate the socket takes is asked. */
|
|
3
|
+
export declare const RECORD_RATE = 16000;
|
|
4
|
+
export declare class PhoneAudio implements DeviceAudio {
|
|
5
|
+
interrupted: boolean;
|
|
6
|
+
private context;
|
|
7
|
+
private queue;
|
|
8
|
+
/** The ids of the buffers enqueued and not yet played. */
|
|
9
|
+
private readonly playing;
|
|
10
|
+
/** A byte of a sample split across two frames. */
|
|
11
|
+
private carry;
|
|
12
|
+
private capture;
|
|
13
|
+
private readonly drained;
|
|
14
|
+
private readonly interruptions;
|
|
15
|
+
private readonly unwatch;
|
|
16
|
+
constructor(options?: AudioOptions);
|
|
17
|
+
prime(): Promise<void>;
|
|
18
|
+
play(pcm: ArrayBuffer, frame: {
|
|
19
|
+
sample_rate: number;
|
|
20
|
+
}): void;
|
|
21
|
+
stop(): void;
|
|
22
|
+
onDrained(fn: () => void): () => void;
|
|
23
|
+
/** At 16 kHz, or at the rate asked when the socket takes it. */
|
|
24
|
+
startCapture(onPcm: (pcm: ArrayBuffer) => void, options?: {
|
|
25
|
+
rate?: number;
|
|
26
|
+
}): Promise<{
|
|
27
|
+
rate: number;
|
|
28
|
+
}>;
|
|
29
|
+
stopCapture(): Promise<void>;
|
|
30
|
+
resume(): Promise<void>;
|
|
31
|
+
setOutput(output: Output): void;
|
|
32
|
+
onInterruption(fn: (interrupted: boolean) => void): () => void;
|
|
33
|
+
dispose(): void;
|
|
34
|
+
/** The queue for a reply at `rate`: this one, or a new context's. */
|
|
35
|
+
private player;
|
|
36
|
+
/**
|
|
37
|
+
* The wire's PCM as floats. **A split sample is carried, never dropped**:
|
|
38
|
+
* the edge sends whole samples (`apps/edge/src/voice/tts.ts`), but an odd
|
|
39
|
+
* byte dropped shifts every byte after it and turns the rest into noise —
|
|
40
|
+
* the bug that once made the console's playback white noise.
|
|
41
|
+
*/
|
|
42
|
+
private samples;
|
|
43
|
+
/** The audio was taken: the reply waits, the recording is thrown away. */
|
|
44
|
+
private interrupt;
|
|
45
|
+
private quiet;
|
|
46
|
+
private tell;
|
|
47
|
+
}
|
|
48
|
+
export declare const createAudio: CreateAudio;
|
|
49
|
+
/** Ask for the microphone, if it was not granted: whether it is now. */
|
|
50
|
+
export declare const requestMicrophone: () => Promise<boolean>;
|
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A phone's player and recorder, on `react-native-audio-api` 0.13.6: what
|
|
3
|
+
* Metro picks for `./audio` on iOS and Android (`index.ts`). With
|
|
4
|
+
* `session.ts`, the only file that touches the library.
|
|
5
|
+
*
|
|
6
|
+
* - **The reply plays through one `AudioBufferQueueSourceNode`**, started
|
|
7
|
+
* once: each frame's PCM is a buffer enqueued behind the last, gapless,
|
|
8
|
+
* and a queue that runs dry plays silence until the next. `clearBuffers`
|
|
9
|
+
* stops it at once.
|
|
10
|
+
* - **The context runs at the reply's rate**, rebuilt — what is queued
|
|
11
|
+
* dropped — when an `audio.start` brings another: 0.13.6's queue plays
|
|
12
|
+
* every buffer at its context's rate whatever the buffer says (its
|
|
13
|
+
* `AudioBufferBaseSourceNode`), so a 24 kHz reply in a 48 kHz context
|
|
14
|
+
* would play an octave up. The device's own resampling does the rest.
|
|
15
|
+
* - **The agent stopped speaking when the last buffer enqueued has ended**,
|
|
16
|
+
* counted by the ids `enqueueBuffer` answers: the queue's own
|
|
17
|
+
* `isLastBufferInQueue` can say so just as the next frame is enqueued, and
|
|
18
|
+
* a buffer `clearBuffers` dropped never ends.
|
|
19
|
+
* - **The microphone records at 16 kHz**, mono, 100 ms a buffer — less to
|
|
20
|
+
* send than 24 kHz, and a rate the socket takes. `onAudioReady` hands the
|
|
21
|
+
* samples over JSI as floats, at the rate the device chose, which need
|
|
22
|
+
* not be the one asked: read from the buffer and resampled (`resample`).
|
|
23
|
+
* - **An interruption** — a call, Siri, another app, the app leaving the
|
|
24
|
+
* screen — suspends the context, so the reply's frames keep queueing and
|
|
25
|
+
* play from where they stopped; and closes the recorder, whose audio is
|
|
26
|
+
* thrown away (the hook discards the recording on the socket). The audio
|
|
27
|
+
* comes back on its own when the system says it may, else on `resume()`
|
|
28
|
+
* or the next press.
|
|
29
|
+
*/
|
|
30
|
+
import { DEFAULT_SAMPLE_RATE, floatToPcm16, pcm16ToFloat, RECORDING_RATES, resample, } from "@lumavox/client";
|
|
31
|
+
import { AudioContext, AudioRecorder, } from "react-native-audio-api";
|
|
32
|
+
import * as session from "./session.js";
|
|
33
|
+
/** Hz the microphone records at unless a rate the socket takes is asked. */
|
|
34
|
+
export const RECORD_RATE = 16000;
|
|
35
|
+
/** Seconds of microphone per buffer delivered: the recorder's own latency. */
|
|
36
|
+
const RECORD_SECONDS = 0.1;
|
|
37
|
+
export class PhoneAudio {
|
|
38
|
+
constructor(options = {}) {
|
|
39
|
+
this.interrupted = false;
|
|
40
|
+
this.context = null;
|
|
41
|
+
this.queue = null;
|
|
42
|
+
/** The ids of the buffers enqueued and not yet played. */
|
|
43
|
+
this.playing = new Set();
|
|
44
|
+
/** A byte of a sample split across two frames. */
|
|
45
|
+
this.carry = null;
|
|
46
|
+
this.capture = null;
|
|
47
|
+
this.drained = new Set();
|
|
48
|
+
this.interruptions = new Set();
|
|
49
|
+
session.configure(options.output ?? "speaker");
|
|
50
|
+
this.unwatch = session.watch(({ began, shouldResume }) => {
|
|
51
|
+
if (began)
|
|
52
|
+
this.interrupt();
|
|
53
|
+
else if (shouldResume && this.interrupted) {
|
|
54
|
+
this.resume().catch(() => {
|
|
55
|
+
// swallowed: still taken — `interrupted` stays, and `resume()` or the next press tries again.
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
async prime() {
|
|
61
|
+
await session.activate();
|
|
62
|
+
this.player(this.context?.sampleRate ?? DEFAULT_SAMPLE_RATE);
|
|
63
|
+
}
|
|
64
|
+
play(pcm, frame) {
|
|
65
|
+
const queue = this.player(frame.sample_rate);
|
|
66
|
+
const samples = this.samples(pcm);
|
|
67
|
+
if (samples.length === 0)
|
|
68
|
+
return;
|
|
69
|
+
const buffer = this.context.createBuffer(1, samples.length, frame.sample_rate);
|
|
70
|
+
buffer.copyToChannel(samples, 0);
|
|
71
|
+
this.playing.add(queue.enqueueBuffer(buffer));
|
|
72
|
+
}
|
|
73
|
+
stop() {
|
|
74
|
+
const was = this.playing.size > 0;
|
|
75
|
+
this.queue?.clearBuffers();
|
|
76
|
+
this.playing.clear();
|
|
77
|
+
// Half a sample of audio that will never play now.
|
|
78
|
+
this.carry = null;
|
|
79
|
+
if (was)
|
|
80
|
+
this.tell(this.drained);
|
|
81
|
+
}
|
|
82
|
+
onDrained(fn) {
|
|
83
|
+
this.drained.add(fn);
|
|
84
|
+
return () => {
|
|
85
|
+
this.drained.delete(fn);
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
/** At 16 kHz, or at the rate asked when the socket takes it. */
|
|
89
|
+
async startCapture(onPcm, options) {
|
|
90
|
+
// A press is the person going on: the audio is taken back first.
|
|
91
|
+
if (this.interrupted)
|
|
92
|
+
await this.resume();
|
|
93
|
+
if (!(await session.microphone())) {
|
|
94
|
+
throw new Error("Microphone permission was refused.");
|
|
95
|
+
}
|
|
96
|
+
// Taken again while the permission was asked: no recording through a call.
|
|
97
|
+
if (this.interrupted)
|
|
98
|
+
throw new Error("The audio was interrupted.");
|
|
99
|
+
const asked = options?.rate;
|
|
100
|
+
const rate = asked && RECORDING_RATES.includes(asked)
|
|
101
|
+
? asked
|
|
102
|
+
: RECORD_RATE;
|
|
103
|
+
const recorder = new AudioRecorder();
|
|
104
|
+
const mine = { recorder, open: true };
|
|
105
|
+
const ready = recorder.onAudioReady({
|
|
106
|
+
sampleRate: rate,
|
|
107
|
+
bufferLength: Math.round(rate * RECORD_SECONDS),
|
|
108
|
+
channelCount: 1,
|
|
109
|
+
}, ({ buffer }) => {
|
|
110
|
+
if (!mine.open)
|
|
111
|
+
return;
|
|
112
|
+
const samples = buffer.getChannelData(0);
|
|
113
|
+
const at = buffer.sampleRate === rate
|
|
114
|
+
? samples
|
|
115
|
+
: resample(samples, buffer.sampleRate, rate);
|
|
116
|
+
onPcm(floatToPcm16(at).buffer);
|
|
117
|
+
});
|
|
118
|
+
if (ready.status === "error")
|
|
119
|
+
throw new Error(ready.message);
|
|
120
|
+
this.capture = mine;
|
|
121
|
+
const started = await recorder.start();
|
|
122
|
+
if (started.status === "error" || !mine.open) {
|
|
123
|
+
if (this.capture === mine)
|
|
124
|
+
this.capture = null;
|
|
125
|
+
mine.open = false;
|
|
126
|
+
recorder.clearOnAudioReady();
|
|
127
|
+
throw new Error(started.status === "error"
|
|
128
|
+
? started.message
|
|
129
|
+
: "The audio was interrupted.");
|
|
130
|
+
}
|
|
131
|
+
return { rate };
|
|
132
|
+
}
|
|
133
|
+
async stopCapture() {
|
|
134
|
+
const mine = this.capture;
|
|
135
|
+
this.capture = null;
|
|
136
|
+
if (!mine)
|
|
137
|
+
return;
|
|
138
|
+
try {
|
|
139
|
+
// What the recorder hands over until it has stopped is the end of
|
|
140
|
+
// the last word: delivered before this settles.
|
|
141
|
+
await mine.recorder.stop();
|
|
142
|
+
}
|
|
143
|
+
finally {
|
|
144
|
+
mine.open = false;
|
|
145
|
+
mine.recorder.clearOnAudioReady();
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
async resume() {
|
|
149
|
+
await session.activate();
|
|
150
|
+
await this.context?.resume();
|
|
151
|
+
if (!this.interrupted)
|
|
152
|
+
return;
|
|
153
|
+
this.interrupted = false;
|
|
154
|
+
this.tell(this.interruptions, false);
|
|
155
|
+
}
|
|
156
|
+
setOutput(output) {
|
|
157
|
+
session.configure(output);
|
|
158
|
+
}
|
|
159
|
+
onInterruption(fn) {
|
|
160
|
+
this.interruptions.add(fn);
|
|
161
|
+
return () => {
|
|
162
|
+
this.interruptions.delete(fn);
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
dispose() {
|
|
166
|
+
this.stop();
|
|
167
|
+
void this.stopCapture();
|
|
168
|
+
this.unwatch();
|
|
169
|
+
void this.context?.close();
|
|
170
|
+
this.context = null;
|
|
171
|
+
this.queue = null;
|
|
172
|
+
}
|
|
173
|
+
/** The queue for a reply at `rate`: this one, or a new context's. */
|
|
174
|
+
player(rate) {
|
|
175
|
+
if (this.queue && this.context?.sampleRate === rate)
|
|
176
|
+
return this.queue;
|
|
177
|
+
this.stop();
|
|
178
|
+
void this.context?.close();
|
|
179
|
+
const context = new AudioContext({ sampleRate: rate });
|
|
180
|
+
const queue = context.createBufferQueueSource();
|
|
181
|
+
queue.connect(context.destination);
|
|
182
|
+
queue.onBufferEnded = ({ bufferId }) => {
|
|
183
|
+
if (this.queue !== queue)
|
|
184
|
+
return;
|
|
185
|
+
if (this.playing.delete(bufferId) && this.playing.size === 0) {
|
|
186
|
+
this.tell(this.drained);
|
|
187
|
+
}
|
|
188
|
+
};
|
|
189
|
+
queue.start();
|
|
190
|
+
// Made while the audio is taken: it waits, as the one it replaced did.
|
|
191
|
+
if (this.interrupted)
|
|
192
|
+
this.quiet(context);
|
|
193
|
+
this.context = context;
|
|
194
|
+
this.queue = queue;
|
|
195
|
+
return queue;
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* The wire's PCM as floats. **A split sample is carried, never dropped**:
|
|
199
|
+
* the edge sends whole samples (`apps/edge/src/voice/tts.ts`), but an odd
|
|
200
|
+
* byte dropped shifts every byte after it and turns the rest into noise —
|
|
201
|
+
* the bug that once made the console's playback white noise.
|
|
202
|
+
*/
|
|
203
|
+
samples(pcm) {
|
|
204
|
+
let bytes = new Uint8Array(pcm);
|
|
205
|
+
if (this.carry !== null) {
|
|
206
|
+
const joined = new Uint8Array(bytes.length + 1);
|
|
207
|
+
joined[0] = this.carry;
|
|
208
|
+
joined.set(bytes, 1);
|
|
209
|
+
bytes = joined;
|
|
210
|
+
this.carry = null;
|
|
211
|
+
}
|
|
212
|
+
if (bytes.length % 2) {
|
|
213
|
+
this.carry = bytes[bytes.length - 1];
|
|
214
|
+
bytes = bytes.slice(0, -1);
|
|
215
|
+
}
|
|
216
|
+
return pcm16ToFloat(bytes.buffer);
|
|
217
|
+
}
|
|
218
|
+
/** The audio was taken: the reply waits, the recording is thrown away. */
|
|
219
|
+
interrupt() {
|
|
220
|
+
if (this.interrupted)
|
|
221
|
+
return;
|
|
222
|
+
this.interrupted = true;
|
|
223
|
+
if (this.context)
|
|
224
|
+
this.quiet(this.context);
|
|
225
|
+
const mine = this.capture;
|
|
226
|
+
this.capture = null;
|
|
227
|
+
if (mine) {
|
|
228
|
+
mine.open = false;
|
|
229
|
+
mine.recorder
|
|
230
|
+
.stop()
|
|
231
|
+
.catch(() => {
|
|
232
|
+
// swallowed: the system has the microphone; it is closed either way.
|
|
233
|
+
})
|
|
234
|
+
.finally(() => mine.recorder.clearOnAudioReady());
|
|
235
|
+
}
|
|
236
|
+
this.tell(this.interruptions, true);
|
|
237
|
+
}
|
|
238
|
+
quiet(context) {
|
|
239
|
+
context.suspend().catch(() => {
|
|
240
|
+
// swallowed: an interruption may have stopped the engine already.
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
tell(listeners, ...args) {
|
|
244
|
+
for (const fn of listeners)
|
|
245
|
+
fn(...args);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
export const createAudio = (options) => new PhoneAudio(options);
|
|
249
|
+
/** Ask for the microphone, if it was not granted: whether it is now. */
|
|
250
|
+
export const requestMicrophone = () => session.microphone();
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import { createWebAudio } from "@lumavox/client/web-audio";
|
|
2
|
+
class BrowserAudio {
|
|
3
|
+
constructor() {
|
|
4
|
+
this.interrupted = false;
|
|
5
|
+
this.web = createWebAudio();
|
|
6
|
+
}
|
|
7
|
+
prime() {
|
|
8
|
+
return this.web.prime();
|
|
9
|
+
}
|
|
10
|
+
play(pcm, frame) {
|
|
11
|
+
this.web.play(pcm, frame);
|
|
12
|
+
}
|
|
13
|
+
stop() {
|
|
14
|
+
this.web.stop();
|
|
15
|
+
}
|
|
16
|
+
onDrained(fn) {
|
|
17
|
+
return this.web.onDrained(fn);
|
|
18
|
+
}
|
|
19
|
+
startCapture(onPcm, options) {
|
|
20
|
+
return this.web.startCapture(onPcm, options);
|
|
21
|
+
}
|
|
22
|
+
stopCapture() {
|
|
23
|
+
return this.web.stopCapture();
|
|
24
|
+
}
|
|
25
|
+
dispose() {
|
|
26
|
+
this.web.dispose();
|
|
27
|
+
}
|
|
28
|
+
resume() {
|
|
29
|
+
return this.web.prime();
|
|
30
|
+
}
|
|
31
|
+
setOutput(_output) { }
|
|
32
|
+
onInterruption(_fn) {
|
|
33
|
+
return () => { };
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
export const createAudio = () => new BrowserAudio();
|
|
37
|
+
/** Ask for the microphone, if it was not granted: whether it is now. */
|
|
38
|
+
export async function requestMicrophone() {
|
|
39
|
+
try {
|
|
40
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
41
|
+
for (const track of stream.getTracks())
|
|
42
|
+
track.stop();
|
|
43
|
+
return true;
|
|
44
|
+
}
|
|
45
|
+
catch {
|
|
46
|
+
// swallowed: a refusal, or no microphone, is the answer — `false`.
|
|
47
|
+
return false;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What `createAudio()` answers on every platform — a phone's
|
|
3
|
+
* (`audio.native.ts`) and a browser's (`audio.web.ts`) — so an app written
|
|
4
|
+
* once runs on both: the `AgentAudio` that `useAgent` drives, and what an
|
|
5
|
+
* app does with the device itself.
|
|
6
|
+
*/
|
|
7
|
+
import type { AgentAudio } from "@lumavox/client";
|
|
8
|
+
/** Where the agent is heard with no headphones on. */
|
|
9
|
+
export type Output = "speaker" | "earpiece";
|
|
10
|
+
export interface AudioOptions {
|
|
11
|
+
/**
|
|
12
|
+
* The loudspeaker (the default) or the earpiece, held to the ear like a
|
|
13
|
+
* call. Headphones and Bluetooth take over when connected, whichever.
|
|
14
|
+
* iPhones only: on Android the loudspeaker until `react-native-audio-api`
|
|
15
|
+
* 1.0 lets the earpiece be chosen.
|
|
16
|
+
*/
|
|
17
|
+
output?: Output;
|
|
18
|
+
}
|
|
19
|
+
export interface DeviceAudio extends AgentAudio {
|
|
20
|
+
/**
|
|
21
|
+
* Get ready to play. **On the web, call it from the press that starts
|
|
22
|
+
* the conversation**: browsers start sound only from a gesture. On a
|
|
23
|
+
* phone it activates the audio session ahead of the first reply.
|
|
24
|
+
*/
|
|
25
|
+
prime(): Promise<void>;
|
|
26
|
+
/** A call, Siri or another app has the audio: playback is paused. */
|
|
27
|
+
readonly interrupted: boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Take the audio back after an interruption that did not give it back
|
|
30
|
+
* itself — the system said not to resume, or another app kept it.
|
|
31
|
+
* Rejects while it is still taken. Pressing to talk does it too.
|
|
32
|
+
*/
|
|
33
|
+
resume(): Promise<void>;
|
|
34
|
+
/** The loudspeaker or the earpiece, from now on (`AudioOptions.output`). */
|
|
35
|
+
setOutput(output: Output): void;
|
|
36
|
+
onInterruption(fn: (interrupted: boolean) => void): () => void;
|
|
37
|
+
}
|
|
38
|
+
/** `createAudio`'s signature, which each platform's file implements. */
|
|
39
|
+
export type CreateAudio = (options?: AudioOptions) => DeviceAudio;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@lumavox/react-native`: one install for a React Native app, on iOS,
|
|
3
|
+
* Android and the web — the client (`createClient`), the hooks
|
|
4
|
+
* (`useAgent` and the reads, `@lumavox/react`), and the audio they lack:
|
|
5
|
+
* `createAudio()`, a player and a recorder for push-to-talk, on
|
|
6
|
+
* `react-native-audio-api` on a phone and Web Audio in a browser.
|
|
7
|
+
*
|
|
8
|
+
* ```ts
|
|
9
|
+
* const audio = useMemo(() => createAudio(), []);
|
|
10
|
+
* const agent = useAgent({ agentId }, { audio });
|
|
11
|
+
* // press: agent.startRecording(); release: agent.stopRecording()
|
|
12
|
+
* ```
|
|
13
|
+
*/
|
|
14
|
+
export * from "@lumavox/client";
|
|
15
|
+
export * from "@lumavox/react";
|
|
16
|
+
export { createAudio, requestMicrophone } from "./audio";
|
|
17
|
+
export type { AudioOptions, CreateAudio, DeviceAudio, Output, } from "./device-audio.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@lumavox/react-native`: one install for a React Native app, on iOS,
|
|
3
|
+
* Android and the web — the client (`createClient`), the hooks
|
|
4
|
+
* (`useAgent` and the reads, `@lumavox/react`), and the audio they lack:
|
|
5
|
+
* `createAudio()`, a player and a recorder for push-to-talk, on
|
|
6
|
+
* `react-native-audio-api` on a phone and Web Audio in a browser.
|
|
7
|
+
*
|
|
8
|
+
* ```ts
|
|
9
|
+
* const audio = useMemo(() => createAudio(), []);
|
|
10
|
+
* const agent = useAgent({ agentId }, { audio });
|
|
11
|
+
* // press: agent.startRecording(); release: agent.stopRecording()
|
|
12
|
+
* ```
|
|
13
|
+
*/
|
|
14
|
+
export * from "@lumavox/client";
|
|
15
|
+
export * from "@lumavox/react";
|
|
16
|
+
// **No extension, on purpose**: Metro picks `audio.native.js` or
|
|
17
|
+
// `audio.web.js` for `./audio` by platform, and only when the import names
|
|
18
|
+
// none — `./audio.js` would be that file alone (`audio.ts`).
|
|
19
|
+
export { createAudio, requestMicrophone } from "./audio";
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { Output } from "./device-audio.js";
|
|
2
|
+
/** The audio session's category, mode and options for `output`. */
|
|
3
|
+
export declare function configure(output: Output): void;
|
|
4
|
+
/** Make the session active: before the first sound, and after an interruption. */
|
|
5
|
+
export declare function activate(): Promise<void>;
|
|
6
|
+
/** Whether the app may record: asked only when not granted yet. */
|
|
7
|
+
export declare function microphone(): Promise<boolean>;
|
|
8
|
+
export interface Interruption {
|
|
9
|
+
/** `true` when the audio is taken, `false` when it is given back. */
|
|
10
|
+
began: boolean;
|
|
11
|
+
/** Given back with the system's word that it may play again. */
|
|
12
|
+
shouldResume: boolean;
|
|
13
|
+
}
|
|
14
|
+
/** Every interruption, the system's and the screen's, until the answer is called. */
|
|
15
|
+
export declare function watch(fn: (event: Interruption) => void): () => void;
|
package/dist/session.js
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The phone's audio session: how the app's audio sits beside the system's
|
|
3
|
+
* — the speaker, the earbuds, calls and other apps. With `audio.native.ts`
|
|
4
|
+
* the only file that touches `react-native-audio-api`, so its 1.0, which
|
|
5
|
+
* renames the session's API (`setSystemOptions`), changes these two and the
|
|
6
|
+
* peer range, and nothing an app sees.
|
|
7
|
+
*
|
|
8
|
+
* - **Play and record at once, from the loudspeaker** (`playAndRecord`,
|
|
9
|
+
* `defaultToSpeaker`): without it iOS plays a recording app's audio from
|
|
10
|
+
* the earpiece. Headphones and Bluetooth take over when connected; the
|
|
11
|
+
* earpiece is the app's choice (`Output`). Android keeps the loudspeaker
|
|
12
|
+
* until 1.0: 0.13.6 sets no communication mode, and so chooses no
|
|
13
|
+
* earpiece.
|
|
14
|
+
* - **The earbuds' microphone on iPhones** (`allowBluetoothHFP`): their
|
|
15
|
+
* hands-free profile, at call quality, for the whole conversation — not
|
|
16
|
+
* switched per press, since every switch moves the route and 0.13.6's
|
|
17
|
+
* engine can die on a Bluetooth route change (its #1209, for R3). Android
|
|
18
|
+
* records from the phone until 1.0, whose `setInputDevice` works.
|
|
19
|
+
* - **`default` mode, not `voiceChat`**: on 0.13.6 `voiceChat` cancels no
|
|
20
|
+
* echo — push-to-talk needs none, as the reply stops while the person
|
|
21
|
+
* speaks — and plays at the call volume, quieter on the loudspeaker.
|
|
22
|
+
* - **Interruptions are watched** (`observeAudioInterruptions`): a call,
|
|
23
|
+
* Siri or another app on iOS, a loss of audio focus on Android — which
|
|
24
|
+
* watching *asks for*, so other apps' audio stops while a conversation is
|
|
25
|
+
* on screen. And **the app leaving the screen** is one: no background
|
|
26
|
+
* audio in this release (the plugin turns the library's off), so the
|
|
27
|
+
* conversation pauses with the screen and goes on when it is back.
|
|
28
|
+
*/
|
|
29
|
+
import { AppState } from "react-native";
|
|
30
|
+
import { AudioManager } from "react-native-audio-api";
|
|
31
|
+
/** The audio session's category, mode and options for `output`. */
|
|
32
|
+
export function configure(output) {
|
|
33
|
+
AudioManager.setAudioSessionOptions({
|
|
34
|
+
iosCategory: "playAndRecord",
|
|
35
|
+
iosMode: "default",
|
|
36
|
+
iosOptions: output === "speaker"
|
|
37
|
+
? ["defaultToSpeaker", "allowBluetoothHFP"]
|
|
38
|
+
: ["allowBluetoothHFP"],
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
/** Make the session active: before the first sound, and after an interruption. */
|
|
42
|
+
export function activate() {
|
|
43
|
+
return AudioManager.setAudioSessionActivity(true);
|
|
44
|
+
}
|
|
45
|
+
/** Whether the app may record: asked only when not granted yet. */
|
|
46
|
+
export async function microphone() {
|
|
47
|
+
if ((await AudioManager.checkRecordingPermissions()) === "Granted") {
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
return (await AudioManager.requestRecordingPermissions()) === "Granted";
|
|
51
|
+
}
|
|
52
|
+
/** Adapters watching: interruptions are observed while any is. */
|
|
53
|
+
let watching = 0;
|
|
54
|
+
/** Every interruption, the system's and the screen's, until the answer is called. */
|
|
55
|
+
export function watch(fn) {
|
|
56
|
+
if (watching++ === 0)
|
|
57
|
+
AudioManager.observeAudioInterruptions(true);
|
|
58
|
+
const system = AudioManager.addSystemEventListener("interruption", (event) => fn({ began: event.type === "began", shouldResume: event.shouldResume }));
|
|
59
|
+
let away = false;
|
|
60
|
+
const screen = AppState.addEventListener("change", (state) => {
|
|
61
|
+
if (state === "background") {
|
|
62
|
+
away = true;
|
|
63
|
+
fn({ began: true, shouldResume: true });
|
|
64
|
+
}
|
|
65
|
+
else if (state === "active" && away) {
|
|
66
|
+
away = false;
|
|
67
|
+
fn({ began: false, shouldResume: true });
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
let done = false;
|
|
71
|
+
return () => {
|
|
72
|
+
if (done)
|
|
73
|
+
return;
|
|
74
|
+
done = true;
|
|
75
|
+
system.remove();
|
|
76
|
+
screen.remove();
|
|
77
|
+
if (--watching === 0)
|
|
78
|
+
AudioManager.observeAudioInterruptions(false);
|
|
79
|
+
};
|
|
80
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,58 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lumavox/react-native",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.9.0",
|
|
4
|
+
"description": "Voice conversations with Lumavox agents in a React Native app, on iOS, Android and the web: the client, the React hooks, and push-to-talk audio on react-native-audio-api.",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"files": [
|
|
7
|
+
"dist",
|
|
8
|
+
"app.plugin.js"
|
|
9
|
+
],
|
|
10
|
+
"sideEffects": false,
|
|
11
|
+
"main": "./dist/index.js",
|
|
12
|
+
"types": "./dist/index.d.ts",
|
|
13
|
+
"exports": {
|
|
14
|
+
".": {
|
|
15
|
+
"@ino/source": "./src/index.ts",
|
|
16
|
+
"types": "./dist/index.d.ts",
|
|
17
|
+
"default": "./dist/index.js"
|
|
18
|
+
},
|
|
19
|
+
"./app.plugin.js": "./app.plugin.js",
|
|
20
|
+
"./package.json": "./package.json"
|
|
21
|
+
},
|
|
22
|
+
"scripts": {
|
|
23
|
+
"build": "node scripts/build.mjs",
|
|
24
|
+
"prepack": "node scripts/build.mjs",
|
|
25
|
+
"check": "biome check --write",
|
|
26
|
+
"format": "biome format --write",
|
|
27
|
+
"lint": "biome lint",
|
|
28
|
+
"test": "vitest run",
|
|
29
|
+
"test:unit": "vitest run",
|
|
30
|
+
"typecheck": "tsc --noEmit && tsc --noEmit -p test"
|
|
31
|
+
},
|
|
32
|
+
"dependencies": {
|
|
33
|
+
"@lumavox/client": "0.9.0",
|
|
34
|
+
"@lumavox/react": "0.9.0"
|
|
35
|
+
},
|
|
36
|
+
"peerDependencies": {
|
|
37
|
+
"react": ">=18",
|
|
38
|
+
"react-native": ">=0.76",
|
|
39
|
+
"react-native-audio-api": ">=0.13.6 <0.14"
|
|
40
|
+
},
|
|
41
|
+
"peerDependenciesMeta": {
|
|
42
|
+
"react-native": {
|
|
43
|
+
"optional": true
|
|
44
|
+
},
|
|
45
|
+
"react-native-audio-api": {
|
|
46
|
+
"optional": true
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
"devDependencies": {
|
|
50
|
+
"@biomejs/biome": "2.6.2",
|
|
51
|
+
"@types/node": "^24",
|
|
52
|
+
"@types/react": "^19",
|
|
53
|
+
"esbuild": "^0.28.1",
|
|
54
|
+
"react": "19.3.1",
|
|
55
|
+
"typescript": "^6",
|
|
56
|
+
"vitest": "^4.1.12"
|
|
57
|
+
}
|
|
58
|
+
}
|