@live-assistant/react-native 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +202 -23
- package/app.plugin.js +7 -0
- package/package.json +7 -6
package/README.md
CHANGED
|
@@ -1,46 +1,225 @@
|
|
|
1
1
|
# @live-assistant/react-native
|
|
2
2
|
|
|
3
|
-
One install for a voice assistant in a React Native or Expo app
|
|
3
|
+
One install for a voice assistant in a React Native or Expo app: the controller,
|
|
4
|
+
a Gemini Live connection, microphone and playback, React bindings and a
|
|
5
|
+
ready-made widget, re-exported from one place.
|
|
6
|
+
|
|
7
|
+
This page is the whole integration. You should not need another one.
|
|
8
|
+
|
|
9
|
+
---
|
|
10
|
+
|
|
11
|
+
## 1. Install
|
|
4
12
|
|
|
5
13
|
```sh
|
|
6
|
-
npm install @live-assistant/react-native
|
|
14
|
+
npm install @live-assistant/react-native react-native-audio-api
|
|
7
15
|
```
|
|
8
16
|
|
|
9
|
-
|
|
17
|
+
`react-native-audio-api` is a **peer dependency and a native module**, so:
|
|
10
18
|
|
|
11
|
-
|
|
19
|
+
- **Expo Go cannot run this.** Build a development build instead:
|
|
20
|
+
`npx expo prebuild && npx expo run:ios` (or `run:android`), or use EAS Build.
|
|
21
|
+
- Installing it means rebuilding the app, not just restarting Metro.
|
|
22
|
+
|
|
23
|
+
On the web nothing native is needed, but `getUserMedia` only exists in a
|
|
24
|
+
**secure context** — serve the page from `https://` or `localhost`.
|
|
25
|
+
|
|
26
|
+
## 2. Configure the microphone — one line
|
|
27
|
+
|
|
28
|
+
```json
|
|
29
|
+
{
|
|
30
|
+
"expo": {
|
|
31
|
+
"plugins": [
|
|
32
|
+
[
|
|
33
|
+
"@live-assistant/react-native",
|
|
34
|
+
{ "microphonePermission": "Acme uses your microphone so you can talk to the assistant." }
|
|
35
|
+
]
|
|
36
|
+
]
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
That is the whole native setup. The plugin ships with the library and writes what
|
|
42
|
+
a voice assistant actually needs — verified by running `expo prebuild` and
|
|
43
|
+
reading the generated files, not the config:
|
|
44
|
+
|
|
45
|
+
| Generated | Value |
|
|
12
46
|
| --- | --- |
|
|
13
|
-
|
|
|
14
|
-
|
|
|
15
|
-
|
|
|
16
|
-
|
|
|
17
|
-
|
|
47
|
+
| `NSMicrophoneUsageDescription` | your sentence — without it iOS terminates the app at the first microphone request |
|
|
48
|
+
| `UIBackgroundModes` | **absent**. `react-native-audio-api`'s own default adds `["audio"]`, which App Review rejects under guideline 2.5.4 when nothing plays in the background |
|
|
49
|
+
| `android.permission.RECORD_AUDIO` | present |
|
|
50
|
+
| foreground service | none |
|
|
51
|
+
|
|
52
|
+
**Do not also list `react-native-audio-api` in `plugins`.** Its plugin runs once,
|
|
53
|
+
so whichever is listed first wins — and if that is theirs, you get the defaults
|
|
54
|
+
this one exists to avoid.
|
|
55
|
+
|
|
56
|
+
Need background audio for real? Configure `react-native-audio-api` yourself
|
|
57
|
+
instead of using this plugin, and be ready to justify the background mode.
|
|
58
|
+
|
|
59
|
+
**Without Expo config plugins** (a bare React Native app), add
|
|
60
|
+
`NSMicrophoneUsageDescription` to `ios/<App>/Info.plist` by hand. `RECORD_AUDIO`
|
|
61
|
+
arrives through the module's own manifest merge on Android.
|
|
62
|
+
|
|
63
|
+
## 3. Mint tokens on your server
|
|
64
|
+
|
|
65
|
+
Your Gemini API key must never ship in an app bundle. Install
|
|
66
|
+
[`@live-assistant/token-server`](https://www.npmjs.com/package/@live-assistant/token-server)
|
|
67
|
+
on your server and put the minting behind your own authentication:
|
|
18
68
|
|
|
19
69
|
```ts
|
|
20
|
-
import {
|
|
70
|
+
import { mintGeminiLiveToken } from '@live-assistant/token-server';
|
|
71
|
+
|
|
72
|
+
app.post('/assistant/token', requireUser, async (req, res) => {
|
|
73
|
+
const minted = await mintGeminiLiveToken({
|
|
74
|
+
apiKey: process.env.GEMINI_API_KEY!,
|
|
75
|
+
model: 'models/gemini-3.1-flash-live-preview',
|
|
76
|
+
systemInstruction: 'You are the assistant inside Acme Notes. Be brief.',
|
|
77
|
+
tools: toolDefinitions, // the same definitions the app registers handlers for
|
|
78
|
+
voiceName: 'Aoede',
|
|
79
|
+
languageCode: req.body.languageCode ?? 'en-US',
|
|
80
|
+
resumptionHandle: req.body.resumptionHandle,
|
|
81
|
+
});
|
|
82
|
+
if (!minted.ok) return res.status(503).json({ error: minted.failure.code });
|
|
83
|
+
res.json(minted.value); // { token, model, wsUrl, expiresAt }
|
|
84
|
+
});
|
|
21
85
|
```
|
|
22
86
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
87
|
+
**The session's configuration lives in the token.** Gemini fixes the instruction,
|
|
88
|
+
the tools and the voice when the token is minted, and discards a setup sent by
|
|
89
|
+
the client — so a tool the token did not declare simply does not exist, with no
|
|
90
|
+
error. Declare every tool here.
|
|
91
|
+
|
|
92
|
+
## 4. Wire up the app
|
|
93
|
+
|
|
94
|
+
```tsx
|
|
95
|
+
import {
|
|
96
|
+
AssistantController,
|
|
97
|
+
AssistantProvider,
|
|
98
|
+
AssistantWidget,
|
|
99
|
+
GeminiLiveSession,
|
|
100
|
+
Microphone,
|
|
101
|
+
PcmPlayer,
|
|
102
|
+
ToolRegistry,
|
|
103
|
+
} from '@live-assistant/react-native';
|
|
26
104
|
|
|
27
|
-
|
|
105
|
+
const tools = new ToolRegistry([
|
|
106
|
+
{
|
|
107
|
+
definition: {
|
|
108
|
+
name: 'createNote',
|
|
109
|
+
description: 'Creates a note with the given text',
|
|
110
|
+
parameters: {
|
|
111
|
+
type: 'object',
|
|
112
|
+
properties: { text: { type: 'string' } },
|
|
113
|
+
required: ['text'],
|
|
114
|
+
},
|
|
115
|
+
},
|
|
116
|
+
run: async ({ text }) => ({ ok: true, id: await notes.create(String(text)) }),
|
|
117
|
+
},
|
|
118
|
+
]);
|
|
28
119
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
120
|
+
const assistant = new AssistantController({
|
|
121
|
+
session: new GeminiLiveSession(),
|
|
122
|
+
microphone: new Microphone(),
|
|
123
|
+
player: new PcmPlayer(),
|
|
124
|
+
tools,
|
|
125
|
+
getConnection: async ({ resumptionHandle }) => {
|
|
126
|
+
const response = await fetch('https://api.example.com/assistant/token', {
|
|
127
|
+
method: 'POST',
|
|
128
|
+
headers: { 'content-type': 'application/json', authorization: `Bearer ${await getUserToken()}` },
|
|
129
|
+
body: JSON.stringify({ resumptionHandle, languageCode: 'en-US' }),
|
|
130
|
+
});
|
|
131
|
+
if (!response.ok) throw await response.json(); // comes back to you as failure.cause
|
|
132
|
+
return response.json();
|
|
133
|
+
},
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
export function App() {
|
|
137
|
+
return (
|
|
138
|
+
<AssistantProvider controller={assistant}>
|
|
139
|
+
<Navigation />
|
|
140
|
+
<AssistantWidget />
|
|
141
|
+
</AssistantProvider>
|
|
142
|
+
);
|
|
143
|
+
}
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Build the controller **once**, outside the component (or in `useState(() => …)`),
|
|
147
|
+
and render the widget near the root so it survives navigation.
|
|
148
|
+
|
|
149
|
+
## 5. Run it
|
|
32
150
|
|
|
33
151
|
```sh
|
|
34
|
-
|
|
152
|
+
npx expo run:ios # or run:android — a development build, not Expo Go
|
|
153
|
+
npx expo start --web # the web half needs no rebuild
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
Tap the orb. It asks for the microphone before spending a token, so the first run
|
|
157
|
+
shows the permission prompt.
|
|
158
|
+
|
|
159
|
+
---
|
|
160
|
+
|
|
161
|
+
## Making it yours
|
|
162
|
+
|
|
163
|
+
```tsx
|
|
164
|
+
<AssistantWidget
|
|
165
|
+
placement="bottom-left"
|
|
166
|
+
theme={{ colors: { primary: '#E4572E', assistantGlow: '#FFB400' }, radius: 8 }}
|
|
167
|
+
strings={{ status: { listening: 'Dinliyorum' }, stop: 'Bitir' }}
|
|
168
|
+
renderTool={(entry, fallback) =>
|
|
169
|
+
entry.status === 'succeeded' ? <ActionChip name={entry.call.name} /> : fallback
|
|
170
|
+
}
|
|
171
|
+
/>
|
|
35
172
|
```
|
|
36
173
|
|
|
37
|
-
|
|
174
|
+
- **`theme`** — colours, orb size, radius, spacing, font size, panel height.
|
|
175
|
+
- **`strings`** — every word, including statuses, errors and end reasons. The
|
|
176
|
+
defaults are English.
|
|
177
|
+
- **`renderMessage` / `renderTool`** — each transcript row, with the default
|
|
178
|
+
rendering handed to you as `fallback`. Note that the default chip shows
|
|
179
|
+
**nothing** for a tool run that succeeded, on the grounds that the assistant
|
|
180
|
+
already said what it did; override `renderTool` to show every run.
|
|
38
181
|
|
|
39
|
-
|
|
40
|
-
|
|
182
|
+
Drawing your own UI instead? Use the hooks — `useAssistant()`,
|
|
183
|
+
`useTranscript()`, `useLevelFrames()` — and install `core`, `gemini`, `audio` and
|
|
184
|
+
`react` directly rather than this package. **It matters for size**: this package
|
|
185
|
+
re-exports with `export *` from a CommonJS build and Metro does not tree-shake,
|
|
186
|
+
so importing one name from it pulls in all five members. Measured on an Expo web
|
|
187
|
+
export whose only import is `AssistantController` — 604 KB through this package
|
|
188
|
+
against 344 KB through `@live-assistant/core`.
|
|
189
|
+
|
|
190
|
+
## What it re-exports
|
|
191
|
+
|
|
192
|
+
| Package | What it brings |
|
|
193
|
+
| --- | --- |
|
|
194
|
+
| `@live-assistant/core` | the controller, the session port, the transcript, tools, levels |
|
|
195
|
+
| `@live-assistant/gemini` | the Gemini Live connection |
|
|
196
|
+
| `@live-assistant/audio` | microphone capture and streaming playback |
|
|
197
|
+
| `@live-assistant/react` | the provider and the hooks |
|
|
198
|
+
| `@live-assistant/widget` | the orb, the panel and the controls |
|
|
199
|
+
|
|
200
|
+
`@live-assistant/token-server` is deliberately **not** here: it mints
|
|
201
|
+
credentials with your API key, so it belongs on a server and never inside an app
|
|
202
|
+
bundle.
|
|
203
|
+
|
|
204
|
+
Node cannot `require` this package — it reaches React Native, whose source is
|
|
205
|
+
Flow. That is expected, and the same reason the token server is separate.
|
|
206
|
+
|
|
207
|
+
## When it does not work
|
|
208
|
+
|
|
209
|
+
| What you see | What it is |
|
|
210
|
+
| --- | --- |
|
|
211
|
+
| `microphone_denied` | The user declined, or `NSMicrophoneUsageDescription` is missing so iOS never asked. Check the **generated** `Info.plist`, not `app.json` |
|
|
212
|
+
| `microphone_unavailable` on a device | The native module is not in the binary — you are on Expo Go. Make a development build |
|
|
213
|
+
| `microphone_unavailable` in a browser | Not a secure context: `getUserMedia` needs `https://` or `localhost` |
|
|
214
|
+
| `connection_refused` | Your `getConnection` threw; the thrown value is on `failure.cause` |
|
|
215
|
+
| `closed_before_ready` | Gemini closed during the handshake — almost always a token used twice. They are single-use |
|
|
216
|
+
| `connect_timed_out` | Check the model you minted with is callable for your key; a model can be listed and still not exist |
|
|
217
|
+
| `no_answer` | The tools declared at mint time do not match what the app registered |
|
|
218
|
+
| Android echoes | Expected: Android's recorder has no echo cancellation, so the controller holds the microphone shut while the assistant is audible |
|
|
219
|
+
| On the web, `start()` never settles | The session was started outside a user gesture. A browser leaves `AudioContext.resume()` pending until the page has been interacted with, so start from a press — which is what the orb already is |
|
|
41
220
|
|
|
42
|
-
|
|
43
|
-
|
|
221
|
+
Failures are codes, never sentences. Map them to words yourself — the library
|
|
222
|
+
ships no user-facing copy.
|
|
44
223
|
|
|
45
224
|
## Licence
|
|
46
225
|
|
package/app.plugin.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The same native configuration as `@live-assistant/audio`, under the name an
|
|
3
|
+
* app already installed — so the plugin list reads like the dependency list.
|
|
4
|
+
*
|
|
5
|
+
* "plugins": [["@live-assistant/react-native", { "microphonePermission": "…" }]]
|
|
6
|
+
*/
|
|
7
|
+
module.exports = require('@live-assistant/audio/app.plugin');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@live-assistant/react-native",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "One install for a voice assistant in a React Native or Expo app: the session, a Gemini Live connection, microphone and playback, React bindings and a ready-made widget, all re-exported from one place.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Recep Tayyip Ekşi",
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
"src",
|
|
29
29
|
"README.md",
|
|
30
30
|
"LICENSE",
|
|
31
|
+
"app.plugin.js",
|
|
31
32
|
"!src/**/__tests__",
|
|
32
33
|
"!src/**/__fixtures__"
|
|
33
34
|
],
|
|
@@ -40,11 +41,11 @@
|
|
|
40
41
|
"access": "public"
|
|
41
42
|
},
|
|
42
43
|
"dependencies": {
|
|
43
|
-
"@live-assistant/audio": "0.
|
|
44
|
-
"@live-assistant/core": "0.
|
|
45
|
-
"@live-assistant/gemini": "0.
|
|
46
|
-
"@live-assistant/react": "0.
|
|
47
|
-
"@live-assistant/widget": "0.
|
|
44
|
+
"@live-assistant/audio": "0.2.0",
|
|
45
|
+
"@live-assistant/core": "0.2.0",
|
|
46
|
+
"@live-assistant/gemini": "0.2.0",
|
|
47
|
+
"@live-assistant/react": "0.2.0",
|
|
48
|
+
"@live-assistant/widget": "0.2.0"
|
|
48
49
|
},
|
|
49
50
|
"peerDependencies": {
|
|
50
51
|
"react": ">=18",
|