@live-assistant/react-native 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +359 -23
- package/app.plugin.js +7 -0
- package/dist/index.d.ts +9 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -4
- package/dist/index.js.map +1 -1
- package/dist/live-assistant.d.ts +67 -0
- package/dist/live-assistant.d.ts.map +1 -0
- package/dist/live-assistant.js +84 -0
- package/dist/live-assistant.js.map +1 -0
- package/package.json +7 -6
- package/src/index.ts +9 -4
- package/src/live-assistant.tsx +137 -0
package/README.md
CHANGED
|
@@ -1,46 +1,382 @@
|
|
|
1
1
|
# @live-assistant/react-native
|
|
2
2
|
|
|
3
|
-
One install for a voice assistant in a React Native or
|
|
3
|
+
One install for a voice assistant in a React Native, Expo or web app — and one
|
|
4
|
+
component to use it:
|
|
5
|
+
|
|
6
|
+
```tsx
|
|
7
|
+
import { LiveAssistant } from '@live-assistant/react-native';
|
|
8
|
+
|
|
9
|
+
<LiveAssistant tokenEndpoint="https://api.example.com/assistant/token" />
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
That is the integration. The Gemini session, the microphone, the player, the tool
|
|
13
|
+
registry, the controller, the provider and the widget are built inside it, the
|
|
14
|
+
same way in every app — so they are not yours to write. The one value it cannot
|
|
15
|
+
invent is the route on **your** server that mints a short-lived token, because
|
|
16
|
+
your API key must never ship in an app bundle.
|
|
17
|
+
|
|
18
|
+
This page is the whole of it: the four steps below, then a table for every value
|
|
19
|
+
you can pass. If you would rather read code than prose,
|
|
20
|
+
[`examples/expo-app`](https://github.com/Recipely-Team/live-assistant/tree/main/examples/expo-app)
|
|
21
|
+
is a working app in one file, and CI builds it on every change.
|
|
22
|
+
|
|
23
|
+
---
|
|
24
|
+
|
|
25
|
+
## 1. Install
|
|
4
26
|
|
|
5
27
|
```sh
|
|
6
|
-
npm install @live-assistant/react-native
|
|
28
|
+
npm install @live-assistant/react-native react-native-audio-api
|
|
7
29
|
```
|
|
8
30
|
|
|
9
|
-
|
|
31
|
+
`react-native-audio-api` is a **peer dependency and a native module**, so:
|
|
10
32
|
|
|
11
|
-
|
|
33
|
+
- **Expo Go cannot run this.** Build a development build instead:
|
|
34
|
+
`npx expo prebuild && npx expo run:ios` (or `run:android`), or use EAS Build.
|
|
35
|
+
- Installing it means rebuilding the app, not just restarting Metro.
|
|
36
|
+
|
|
37
|
+
On the web nothing native is needed, but `getUserMedia` only exists in a
|
|
38
|
+
**secure context** — serve the page from `https://` or `localhost`.
|
|
39
|
+
|
|
40
|
+
## 2. Configure the microphone — one line
|
|
41
|
+
|
|
42
|
+
```json
|
|
43
|
+
{
|
|
44
|
+
"expo": {
|
|
45
|
+
"plugins": [
|
|
46
|
+
[
|
|
47
|
+
"@live-assistant/react-native",
|
|
48
|
+
{ "microphonePermission": "Acme uses your microphone so you can talk to the assistant." }
|
|
49
|
+
]
|
|
50
|
+
]
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
That is the whole native setup. The plugin ships with the library and writes what
|
|
56
|
+
a voice assistant actually needs — verified by running `expo prebuild` and
|
|
57
|
+
reading the generated files, not the config:
|
|
58
|
+
|
|
59
|
+
| Generated | Value |
|
|
12
60
|
| --- | --- |
|
|
13
|
-
|
|
|
14
|
-
|
|
|
15
|
-
|
|
|
16
|
-
|
|
|
17
|
-
|
|
61
|
+
| `NSMicrophoneUsageDescription` | your sentence — without it iOS terminates the app at the first microphone request |
|
|
62
|
+
| `UIBackgroundModes` | **absent**. `react-native-audio-api`'s own default adds `["audio"]`, which App Review rejects under guideline 2.5.4 when nothing plays in the background |
|
|
63
|
+
| `android.permission.RECORD_AUDIO` | present |
|
|
64
|
+
| foreground service | none |
|
|
65
|
+
|
|
66
|
+
**Do not also list `react-native-audio-api` in `plugins`.** Its plugin runs once,
|
|
67
|
+
so whichever is listed first wins — and if that is theirs, you get the defaults
|
|
68
|
+
this one exists to avoid.
|
|
69
|
+
|
|
70
|
+
Need background audio for real? Configure `react-native-audio-api` yourself
|
|
71
|
+
instead of using this plugin, and be ready to justify the background mode.
|
|
72
|
+
|
|
73
|
+
**Without Expo config plugins** (a bare React Native app), add
|
|
74
|
+
`NSMicrophoneUsageDescription` to `ios/<App>/Info.plist` by hand. `RECORD_AUDIO`
|
|
75
|
+
arrives through the module's own manifest merge on Android.
|
|
76
|
+
|
|
77
|
+
## 3. Mint tokens on your server
|
|
78
|
+
|
|
79
|
+
Your Gemini API key must never ship in an app bundle. Install
|
|
80
|
+
[`@live-assistant/token-server`](https://www.npmjs.com/package/@live-assistant/token-server)
|
|
81
|
+
on your server and put the minting behind your own authentication:
|
|
18
82
|
|
|
19
83
|
```ts
|
|
20
|
-
import {
|
|
84
|
+
import { mintGeminiLiveToken } from '@live-assistant/token-server';
|
|
85
|
+
|
|
86
|
+
app.post('/assistant/token', requireUser, async (req, res) => {
|
|
87
|
+
const minted = await mintGeminiLiveToken({
|
|
88
|
+
apiKey: process.env.GEMINI_API_KEY!,
|
|
89
|
+
model: 'models/gemini-3.1-flash-live-preview',
|
|
90
|
+
systemInstruction: 'You are the assistant inside Acme Notes. Be brief.',
|
|
91
|
+
tools: toolDefinitions, // the same definitions the app registers handlers for
|
|
92
|
+
voiceName: 'Aoede',
|
|
93
|
+
languageCode: req.body.languageCode ?? 'en-US',
|
|
94
|
+
resumptionHandle: req.body.resumptionHandle,
|
|
95
|
+
});
|
|
96
|
+
if (!minted.ok) return res.status(503).json({ error: minted.failure.code });
|
|
97
|
+
res.json(minted.value); // { token, model, wsUrl, expiresAt }
|
|
98
|
+
});
|
|
21
99
|
```
|
|
22
100
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
101
|
+
**The session's configuration lives in the token.** Gemini fixes the instruction,
|
|
102
|
+
the tools and the voice when the token is minted, and discards a setup sent by
|
|
103
|
+
the client — so a tool the token did not declare simply does not exist, with no
|
|
104
|
+
error. Declare every tool here.
|
|
105
|
+
|
|
106
|
+
## 4. Add the assistant
|
|
107
|
+
|
|
108
|
+
```tsx
|
|
109
|
+
import { LiveAssistant } from '@live-assistant/react-native';
|
|
110
|
+
|
|
111
|
+
export function App() {
|
|
112
|
+
return (
|
|
113
|
+
<>
|
|
114
|
+
<Navigation />
|
|
115
|
+
<LiveAssistant
|
|
116
|
+
tokenEndpoint="https://api.example.com/assistant/token"
|
|
117
|
+
headers={{ authorization: `Bearer ${userToken}` }}
|
|
118
|
+
/>
|
|
119
|
+
</>
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Render it once, near the root and beside your navigation, so a session survives
|
|
125
|
+
moving between screens. It builds the controller on its first render and stops
|
|
126
|
+
the session when it unmounts; `headers` and `language` are read again on every
|
|
127
|
+
connection, so a login refreshed mid-conversation is the one used when a long
|
|
128
|
+
session hands over to a new socket.
|
|
129
|
+
|
|
130
|
+
### What it can already do, with nothing registered
|
|
131
|
+
|
|
132
|
+
On the web the assistant reads the page it is on and acts on it:
|
|
26
133
|
|
|
27
|
-
|
|
134
|
+
| It can | Meaning |
|
|
135
|
+
| --- | --- |
|
|
136
|
+
| `read` | what the page says, as text |
|
|
137
|
+
| `list` | what can be followed, pressed or filled, by name |
|
|
138
|
+
| `navigate` | follow a link, by its name or a path |
|
|
139
|
+
| `back` | go back |
|
|
140
|
+
| `press` | click a button or link |
|
|
141
|
+
| `type` | put text in a field |
|
|
142
|
+
| `scroll` | up, down, to the top or the bottom |
|
|
143
|
+
|
|
144
|
+
Nothing declares any of that. It reads the live DOM at the moment of the call, so
|
|
145
|
+
there is no route table to keep in step with your router and nothing to register
|
|
146
|
+
on a new screen. Targets are named the way someone reading the screen aloud would
|
|
147
|
+
name them — the accessible name — which is also why a React Native Web app works
|
|
148
|
+
unchanged: `accessibilityLabel` renders as `aria-label`.
|
|
149
|
+
|
|
150
|
+
Following a link **clicks** it rather than assigning the url, so your
|
|
151
|
+
single-page router stays in charge and the live session is not thrown away by a
|
|
152
|
+
reload. A target that is not on the page is answered with the names that are, so
|
|
153
|
+
the model picks one instead of guessing again.
|
|
154
|
+
|
|
155
|
+
On a phone there is no DOM, so the pack is inert unless you hand it a router:
|
|
156
|
+
|
|
157
|
+
```tsx
|
|
158
|
+
<LiveAssistant
|
|
159
|
+
tokenEndpoint={endpoint}
|
|
160
|
+
page={{ router: { go: (path) => router.push(path), back: () => router.back(), current: () => pathname } }}
|
|
161
|
+
/>
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
### Your own tools
|
|
165
|
+
|
|
166
|
+
Everything the page cannot do for itself is a tool:
|
|
167
|
+
|
|
168
|
+
```tsx
|
|
169
|
+
<LiveAssistant
|
|
170
|
+
tokenEndpoint={endpoint}
|
|
171
|
+
tools={[
|
|
172
|
+
{
|
|
173
|
+
definition: {
|
|
174
|
+
name: 'createNote',
|
|
175
|
+
description: 'Creates a note with the given text',
|
|
176
|
+
parameters: { type: 'object', properties: { text: { type: 'string' } }, required: ['text'] },
|
|
177
|
+
},
|
|
178
|
+
run: async ({ text }) => ({ ok: true, id: await notes.create(String(text)) }),
|
|
179
|
+
},
|
|
180
|
+
]}
|
|
181
|
+
/>
|
|
182
|
+
```
|
|
28
183
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
184
|
+
Declare the same definitions when you mint the token (step 3): Gemini fixes the
|
|
185
|
+
tool list there, and a tool the token did not declare does not exist — with no
|
|
186
|
+
error saying so.
|
|
187
|
+
|
|
188
|
+
## 5. Run it
|
|
32
189
|
|
|
33
190
|
```sh
|
|
34
|
-
|
|
191
|
+
npx expo run:ios # or run:android — a development build, not Expo Go
|
|
192
|
+
npx expo start --web # the web half needs no rebuild
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Tap the orb. It asks for the microphone before spending a token, so the first run
|
|
196
|
+
shows the permission prompt.
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
## Configuration
|
|
201
|
+
|
|
202
|
+
Everything is optional except `tokenEndpoint` (or `getConnection` in its place).
|
|
203
|
+
|
|
204
|
+
### `<LiveAssistant>`
|
|
205
|
+
|
|
206
|
+
| Prop | Default | What it does |
|
|
207
|
+
| --- | --- | --- |
|
|
208
|
+
| **`tokenEndpoint`** | — | **Required.** Your server's route that mints a token. Called as `POST` with `{ resumptionHandle, languageCode }`; answer `{ token, model, wsUrl? }` |
|
|
209
|
+
| `getConnection` | — | Instead of `tokenEndpoint`, when your app already has its own client. Throw to refuse — the thrown value comes back as `failure.cause` |
|
|
210
|
+
| `headers` | — | Added to the token request; where your `Authorization` goes. Read fresh on every connection |
|
|
211
|
+
| `language` | `'en-US'` | Sent to your endpoint as `languageCode` |
|
|
212
|
+
| `tools` | — | Your app's own tools, on top of the page pack. An array or a `ToolRegistry` |
|
|
213
|
+
| `page` | on where a document exists | Reading and driving the page. `false` removes it; an object narrows it — table below |
|
|
214
|
+
| `timing` | measured | `utteranceGapMs` 1200 · `answerTimeoutMs` 12000 · `silenceTimeoutMs` 90000, `null` never ends a session · `echoTailMs` 250 · `maxHandovers` 3 |
|
|
215
|
+
| `theme` | below | Colours, sizes and your logo |
|
|
216
|
+
| `strings` | English | Every word the widget says |
|
|
217
|
+
| `placement` | `'bottom-right'` | `'bottom-left'`, or `'inline'` to lay it out where you rendered it |
|
|
218
|
+
| `style` | — | Merged last onto the floating stack — safe-area insets go here (`{ bottom: insets.bottom + 16 }`) |
|
|
219
|
+
| `showTranscript` | `true` | Show the conversation in the panel |
|
|
220
|
+
| `showComposer` | `true` | Show the typing box |
|
|
221
|
+
| `renderMessage` | — | Replace a said line; the default bubble arrives as `fallback` |
|
|
222
|
+
| `renderTool` | — | Replace a tool run; return `null` to hide it. The default shows **nothing** for a run that succeeded, on the grounds that the assistant already said what it did |
|
|
223
|
+
| `onReady` | — | Called once with the controller, to start or stop a session from elsewhere (a push notification, a deep link) |
|
|
224
|
+
| `onFailure` | — | Called with each failure, for logging. The widget already tells the user |
|
|
225
|
+
|
|
226
|
+
### `page`
|
|
227
|
+
|
|
228
|
+
| Field | Default | What it does |
|
|
229
|
+
| --- | --- | --- |
|
|
230
|
+
| `actions` | all seven | Which actions the model may use. A word left out never reaches the model, so it cannot ask for it and be refused |
|
|
231
|
+
| `name` | `'page'` | The tool's name, if `page` collides with one of yours |
|
|
232
|
+
| `root` | the whole document | A CSS selector the tools are confined to |
|
|
233
|
+
| `maxCharacters` | `4000` | Cap on what `read` returns, so a long page cannot crowd out the conversation |
|
|
234
|
+
| `maxTargets` | `40` | Cap on what `list` returns per kind |
|
|
235
|
+
| `router` | — | `{ go, back, current }` — navigation where there is no DOM |
|
|
236
|
+
| `document` / `window` | the globals | For tests, an iframe, or a server render |
|
|
237
|
+
|
|
238
|
+
### `theme`
|
|
239
|
+
|
|
240
|
+
Pass any part of it; the rest keeps the default.
|
|
241
|
+
|
|
242
|
+
| Colour | Default | Where it shows |
|
|
243
|
+
| --- | --- | --- |
|
|
244
|
+
| `primary` | `#5B5BD6` | The orb at rest, and the controls' accent |
|
|
245
|
+
| `userGlow` | `#3E9BFF` | The ring that follows the user's voice |
|
|
246
|
+
| `assistantGlow` | `#B45BFF` | The glow that follows the assistant's voice |
|
|
247
|
+
| `surface` | `#FFFFFF` | The panel |
|
|
248
|
+
| `text` | `#1C1C28` | Panel text |
|
|
249
|
+
| `mutedText` | `#6B6B80` | The status line and secondary text |
|
|
250
|
+
| `userBubble` | `#5B5BD6` | The user's transcript bubble |
|
|
251
|
+
| `userText` | `#FFFFFF` | Text in it |
|
|
252
|
+
| `assistantBubble` | `#F0F0F7` | The assistant's bubble |
|
|
253
|
+
| `assistantText` | `#1C1C28` | Text in it |
|
|
254
|
+
| `toolChip` | `#E8F5EC` | A tool-run chip |
|
|
255
|
+
| `toolText` | `#1F6B3A` | Text on it |
|
|
256
|
+
| `danger` | `#D93F3F` | Errors, and the end-session control |
|
|
257
|
+
| `onPrimary` | `#FFFFFF` | Anything drawn on `primary` |
|
|
258
|
+
|
|
259
|
+
| Value | Default | What it does |
|
|
260
|
+
| --- | --- | --- |
|
|
261
|
+
| `logo` | — | **Your mark, drawn inside the orb.** Any `<Image source>`: a `require(...)`, a `{ uri }`, an imported asset |
|
|
262
|
+
| `logoSize` | `0.55` | The logo's share of the orb's diameter; `1` would touch the edges |
|
|
263
|
+
| `orbSize` | `64` | The orb's diameter |
|
|
264
|
+
| `radius` | `16` | Corner radius of the panel and its controls |
|
|
265
|
+
| `spacing` | `12` | Padding and the gaps between rows |
|
|
266
|
+
| `fontSize` | `15` | Transcript text size |
|
|
267
|
+
| `panelMaxHeight` | `420` | How tall the panel may grow before the transcript scrolls |
|
|
268
|
+
|
|
269
|
+
```tsx
|
|
270
|
+
<LiveAssistant
|
|
271
|
+
tokenEndpoint={endpoint}
|
|
272
|
+
theme={{ logo: require('./assets/mark.png'), colors: { primary: '#E4572E', assistantGlow: '#FFB400' }, radius: 8 }}
|
|
273
|
+
/>
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
### `strings`
|
|
277
|
+
|
|
278
|
+
Every word, in one object — the defaults are English and the library ships no
|
|
279
|
+
other language, because a library that guesses at your voice is one you have to
|
|
280
|
+
argue with.
|
|
281
|
+
|
|
282
|
+
| Key | Default |
|
|
283
|
+
| --- | --- |
|
|
284
|
+
| `start` / `stop` | `Start voice assistant` / `End` |
|
|
285
|
+
| `mute` / `unmute` | `Mute` / `Unmute` |
|
|
286
|
+
| `send` / `composerPlaceholder` | `Send` / `Type a message` |
|
|
287
|
+
| `status.idle` … `status.working` | `Tap to talk`, `Connecting…`, `Listening`, `Thinking…`, `Speaking`, `Working on it…` |
|
|
288
|
+
| `ended.silence` | `Ended after a quiet spell` |
|
|
289
|
+
| `errors.*` | one sentence per failure code — `microphone_denied`, `microphone_unavailable`, `connection_refused`, `connection_lost`, `no_answer` |
|
|
290
|
+
| `genericError` | `Something went wrong` |
|
|
291
|
+
| `toolRunning(name)` / `toolFailed(name)` | `Running {name}…` / `{name} did not work` |
|
|
292
|
+
|
|
293
|
+
```tsx
|
|
294
|
+
<LiveAssistant
|
|
295
|
+
tokenEndpoint={endpoint}
|
|
296
|
+
strings={{ start: 'Asistanı başlat', stop: 'Bitir', status: { listening: 'Dinliyorum' } }}
|
|
297
|
+
/>
|
|
35
298
|
```
|
|
36
299
|
|
|
37
|
-
|
|
300
|
+
---
|
|
38
301
|
|
|
39
|
-
|
|
40
|
-
|
|
302
|
+
## Building it yourself
|
|
303
|
+
|
|
304
|
+
`<LiveAssistant>` is not a wall. It is these three things, and nothing you cannot
|
|
305
|
+
write out when you need to hold them apart — a controller that outlives the tree,
|
|
306
|
+
a widget somewhere other than where the provider is, a session you start from a
|
|
307
|
+
push notification:
|
|
308
|
+
|
|
309
|
+
```tsx
|
|
310
|
+
const assistant = new AssistantController({
|
|
311
|
+
session: new GeminiLiveSession(),
|
|
312
|
+
microphone: new Microphone(),
|
|
313
|
+
player: new PcmPlayer(),
|
|
314
|
+
tools: new ToolRegistry([...]),
|
|
315
|
+
getConnection: async ({ resumptionHandle }) => {
|
|
316
|
+
const response = await fetch('https://api.example.com/assistant/token', {
|
|
317
|
+
method: 'POST',
|
|
318
|
+
headers: { 'content-type': 'application/json', authorization: `Bearer ${await getUserToken()}` },
|
|
319
|
+
body: JSON.stringify({ resumptionHandle, languageCode: 'en-US' }),
|
|
320
|
+
});
|
|
321
|
+
if (!response.ok) throw await response.json(); // comes back to you as failure.cause
|
|
322
|
+
return response.json();
|
|
323
|
+
},
|
|
324
|
+
});
|
|
325
|
+
|
|
326
|
+
export function App() {
|
|
327
|
+
return (
|
|
328
|
+
<AssistantProvider controller={assistant}>
|
|
329
|
+
<Navigation />
|
|
330
|
+
<AssistantWidget />
|
|
331
|
+
</AssistantProvider>
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
```
|
|
335
|
+
|
|
336
|
+
Build the controller **once**, outside the component or in `useState(() => …)`:
|
|
337
|
+
it owns a socket and two devices.
|
|
338
|
+
|
|
339
|
+
Drawing your own UI instead of the widget? Use the hooks — `useAssistant()`,
|
|
340
|
+
`useTranscript()`, `useLevelFrames()` — and install `core`, `gemini`, `audio` and
|
|
341
|
+
`react` directly rather than this package. **It matters for size**: this package
|
|
342
|
+
re-exports with `export *` from a CommonJS build and Metro does not tree-shake,
|
|
343
|
+
so importing one name from it pulls in all five members. Measured on an Expo web
|
|
344
|
+
export whose only import is `AssistantController` — 604 KB through this package
|
|
345
|
+
against 344 KB through `@live-assistant/core`.
|
|
346
|
+
|
|
347
|
+
## What it re-exports
|
|
348
|
+
|
|
349
|
+
| Package | What it brings |
|
|
350
|
+
| --- | --- |
|
|
351
|
+
| `@live-assistant/core` | the controller, the session port, the transcript, tools, levels |
|
|
352
|
+
| `@live-assistant/gemini` | the Gemini Live connection |
|
|
353
|
+
| `@live-assistant/audio` | microphone capture and streaming playback |
|
|
354
|
+
| `@live-assistant/react` | the provider and the hooks |
|
|
355
|
+
| `@live-assistant/widget` | the orb, the panel and the controls |
|
|
356
|
+
|
|
357
|
+
`@live-assistant/token-server` is deliberately **not** here: it mints
|
|
358
|
+
credentials with your API key, so it belongs on a server and never inside an app
|
|
359
|
+
bundle.
|
|
360
|
+
|
|
361
|
+
Node cannot `require` this package — it reaches React Native, whose source is
|
|
362
|
+
Flow. That is expected, and the same reason the token server is separate.
|
|
363
|
+
|
|
364
|
+
## When it does not work
|
|
365
|
+
|
|
366
|
+
| What you see | What it is |
|
|
367
|
+
| --- | --- |
|
|
368
|
+
| `microphone_denied` | The user declined, or `NSMicrophoneUsageDescription` is missing so iOS never asked. Check the **generated** `Info.plist`, not `app.json` |
|
|
369
|
+
| `microphone_unavailable` on a device | The native module is not in the binary — you are on Expo Go. Make a development build |
|
|
370
|
+
| `microphone_unavailable` in a browser | Not a secure context: `getUserMedia` needs `https://` or `localhost` |
|
|
371
|
+
| `connection_refused` | Your `getConnection` threw; the thrown value is on `failure.cause` |
|
|
372
|
+
| `closed_before_ready` | Gemini closed during the handshake — almost always a token used twice. They are single-use |
|
|
373
|
+
| `connect_timed_out` | Check the model you minted with is callable for your key; a model can be listed and still not exist |
|
|
374
|
+
| `no_answer` | The tools declared at mint time do not match what the app registered |
|
|
375
|
+
| Android echoes | Expected: Android's recorder has no echo cancellation, so the controller holds the microphone shut while the assistant is audible |
|
|
376
|
+
| On the web, `start()` never settles | The session was started outside a user gesture. A browser leaves `AudioContext.resume()` pending until the page has been interacted with, so start from a press — which is what the orb already is |
|
|
41
377
|
|
|
42
|
-
|
|
43
|
-
|
|
378
|
+
Failures are codes, never sentences. Map them to words yourself — the library
|
|
379
|
+
ships no user-facing copy.
|
|
44
380
|
|
|
45
381
|
## Licence
|
|
46
382
|
|
package/app.plugin.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The same native configuration as `@live-assistant/audio`, under the name an
|
|
3
|
+
* app already installed — so the plugin list reads like the dependency list.
|
|
4
|
+
*
|
|
5
|
+
* "plugins": [["@live-assistant/react-native", { "microphonePermission": "…" }]]
|
|
6
|
+
*/
|
|
7
|
+
module.exports = require('@live-assistant/audio/app.plugin');
|
package/dist/index.d.ts
CHANGED
|
@@ -9,10 +9,13 @@
|
|
|
9
9
|
* should be able to leave out. None of those reasons apply to the ordinary
|
|
10
10
|
* case — an app that wants the assistant — and making that case install five
|
|
11
11
|
* packages and keep five versions in step is friction with nothing behind it.
|
|
12
|
-
* - **It adds
|
|
13
|
-
* so reading the source of
|
|
14
|
-
* are holding, and an app that
|
|
15
|
-
* pieces directly without changing
|
|
12
|
+
* - **It adds exactly one thing of its own: `<LiveAssistant>`.** Everything
|
|
13
|
+
* else is re-exported unchanged, so reading the source of
|
|
14
|
+
* `@live-assistant/core` still explains what you are holding, and an app that
|
|
15
|
+
* outgrows this package can depend on the pieces directly without changing an
|
|
16
|
+
* import. The one addition exists because this is the only package that can
|
|
17
|
+
* see the session, the audio and the widget at once — and because the forty
|
|
18
|
+
* lines it replaces were identical in every app that would ever write them.
|
|
16
19
|
* - **The token server is deliberately absent.** It mints credentials with an
|
|
17
20
|
* API key, which belongs on a server and never in an app bundle. It is
|
|
18
21
|
* installed on its own, where it runs.
|
|
@@ -22,4 +25,6 @@ export * from '@live-assistant/gemini';
|
|
|
22
25
|
export * from '@live-assistant/audio';
|
|
23
26
|
export * from '@live-assistant/react';
|
|
24
27
|
export * from '@live-assistant/widget';
|
|
28
|
+
export { LiveAssistant } from './live-assistant';
|
|
29
|
+
export type { LiveAssistantProps } from './live-assistant';
|
|
25
30
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,YAAY,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -14,6 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
+
exports.LiveAssistant = void 0;
|
|
17
18
|
/**
|
|
18
19
|
* Everything an app on a phone needs, from one install.
|
|
19
20
|
*
|
|
@@ -25,10 +26,13 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
25
26
|
* should be able to leave out. None of those reasons apply to the ordinary
|
|
26
27
|
* case — an app that wants the assistant — and making that case install five
|
|
27
28
|
* packages and keep five versions in step is friction with nothing behind it.
|
|
28
|
-
* - **It adds
|
|
29
|
-
* so reading the source of
|
|
30
|
-
* are holding, and an app that
|
|
31
|
-
* pieces directly without changing
|
|
29
|
+
* - **It adds exactly one thing of its own: `<LiveAssistant>`.** Everything
|
|
30
|
+
* else is re-exported unchanged, so reading the source of
|
|
31
|
+
* `@live-assistant/core` still explains what you are holding, and an app that
|
|
32
|
+
* outgrows this package can depend on the pieces directly without changing an
|
|
33
|
+
* import. The one addition exists because this is the only package that can
|
|
34
|
+
* see the session, the audio and the widget at once — and because the forty
|
|
35
|
+
* lines it replaces were identical in every app that would ever write them.
|
|
32
36
|
* - **The token server is deliberately absent.** It mints credentials with an
|
|
33
37
|
* API key, which belongs on a server and never in an app bundle. It is
|
|
34
38
|
* installed on its own, where it runs.
|
|
@@ -38,4 +42,6 @@ __exportStar(require("@live-assistant/gemini"), exports);
|
|
|
38
42
|
__exportStar(require("@live-assistant/audio"), exports);
|
|
39
43
|
__exportStar(require("@live-assistant/react"), exports);
|
|
40
44
|
__exportStar(require("@live-assistant/widget"), exports);
|
|
45
|
+
var live_assistant_1 = require("./live-assistant");
|
|
46
|
+
Object.defineProperty(exports, "LiveAssistant", { enumerable: true, get: function () { return live_assistant_1.LiveAssistant; } });
|
|
41
47
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,uDAAqC;AACrC,yDAAuC;AACvC,wDAAsC;AACtC,wDAAsC;AACtC,yDAAuC;AACvC,mDAAiD;AAAxC,+GAAA,aAAa,OAAA"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { AssistantController, ToolRegistry } from '@live-assistant/core';
|
|
2
|
+
import type { AssistantControllerOptions, AssistantFailure, AssistantTool, PageToolsOptions } from '@live-assistant/core';
|
|
3
|
+
import type { GeminiLiveCredentials } from '@live-assistant/gemini';
|
|
4
|
+
import type { AssistantWidgetProps } from '@live-assistant/widget';
|
|
5
|
+
/** Shared by both ways of connecting; see `LiveAssistantProps`. */
|
|
6
|
+
interface LiveAssistantCommonProps extends AssistantWidgetProps {
|
|
7
|
+
/** Sent to your token endpoint as `languageCode`. Default `'en-US'`. */
|
|
8
|
+
readonly language?: string;
|
|
9
|
+
/** Added to the token request — this is where an app's own `Authorization` goes. Read fresh on every connection. */
|
|
10
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
11
|
+
/** Your app's own tools, on top of the page pack. */
|
|
12
|
+
readonly tools?: readonly AssistantTool[] | ToolRegistry;
|
|
13
|
+
/** Reading and driving the page. On by default in a browser; `false` removes it. */
|
|
14
|
+
readonly page?: boolean | PageToolsOptions;
|
|
15
|
+
readonly timing?: AssistantControllerOptions<GeminiLiveCredentials>['timing'];
|
|
16
|
+
/** Called with the controller once, for an app that wants to start or stop a session from elsewhere. */
|
|
17
|
+
readonly onReady?: (controller: AssistantController<GeminiLiveCredentials>) => void;
|
|
18
|
+
/** Called whenever a session fails. The widget already tells the user; this is for logging. */
|
|
19
|
+
readonly onFailure?: (failure: AssistantFailure) => void;
|
|
20
|
+
}
|
|
21
|
+
/** The endpoint form: the usual one. */
|
|
22
|
+
interface LiveAssistantEndpointProps extends LiveAssistantCommonProps {
|
|
23
|
+
/** Your server's route that mints a short-lived token. The one value this component cannot invent. */
|
|
24
|
+
readonly tokenEndpoint: string;
|
|
25
|
+
readonly getConnection?: never;
|
|
26
|
+
}
|
|
27
|
+
/** The escape hatch: an app that already has its own client for the same thing. */
|
|
28
|
+
interface LiveAssistantConnectionProps extends LiveAssistantCommonProps {
|
|
29
|
+
readonly getConnection: (request: {
|
|
30
|
+
readonly resumptionHandle?: string;
|
|
31
|
+
}) => Promise<GeminiLiveCredentials>;
|
|
32
|
+
readonly tokenEndpoint?: never;
|
|
33
|
+
}
|
|
34
|
+
export type LiveAssistantProps = LiveAssistantEndpointProps | LiveAssistantConnectionProps;
|
|
35
|
+
/**
|
|
36
|
+
* The whole integration: import it, put it in your tree, and there is a voice
|
|
37
|
+
* assistant on the screen.
|
|
38
|
+
*
|
|
39
|
+
* ```tsx
|
|
40
|
+
* <LiveAssistant tokenEndpoint="https://api.example.com/assistant/token" />
|
|
41
|
+
* ```
|
|
42
|
+
*
|
|
43
|
+
* @remarks
|
|
44
|
+
* - **One required value, because it is the only one we cannot know**: the
|
|
45
|
+
* route on your server that mints a short-lived token. Everything else — the
|
|
46
|
+
* Gemini session, the microphone, the player, the tool registry, the
|
|
47
|
+
* controller, the provider and the widget — is built here, the same way in
|
|
48
|
+
* every app, which is what makes it ours to write rather than yours.
|
|
49
|
+
* - **The controller is built once**, in a `useState` initialiser. It owns a
|
|
50
|
+
* socket and two devices; rebuilding it on a render would start a session per
|
|
51
|
+
* render. The session is stopped when this unmounts.
|
|
52
|
+
* - **`headers` and `language` are read at connection time, not at mount.** An
|
|
53
|
+
* app whose `Authorization` header is refreshed mid-session would otherwise
|
|
54
|
+
* reconnect with the token it had when the component first rendered — which
|
|
55
|
+
* is exactly when a long conversation hands over to a new socket.
|
|
56
|
+
* - **A non-2xx from your endpoint is thrown on purpose.** It reaches the app
|
|
57
|
+
* as `connection_refused` with the response in `failure.cause`, rather than
|
|
58
|
+
* as a session that fails later for no stated reason.
|
|
59
|
+
* - **It does not own your navigation or your screens.** The page pack reads
|
|
60
|
+
* the live DOM, so on the web it already works; a native app passes its own
|
|
61
|
+
* tools, or `page: { router }`.
|
|
62
|
+
* - **Outgrowing it costs nothing**: build `AssistantController` yourself and
|
|
63
|
+
* render `AssistantProvider` + `AssistantWidget`, which is all this does.
|
|
64
|
+
*/
|
|
65
|
+
export declare function LiveAssistant({ tokenEndpoint, getConnection, language, headers, tools, page, timing, onReady, onFailure, ...widget }: LiveAssistantProps): import("react").JSX.Element;
|
|
66
|
+
export {};
|
|
67
|
+
//# sourceMappingURL=live-assistant.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"live-assistant.d.ts","sourceRoot":"","sources":["../src/live-assistant.tsx"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzE,OAAO,KAAK,EAAE,0BAA0B,EAAE,gBAAgB,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAG1H,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAGpE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAEnE,mEAAmE;AACnE,UAAU,wBAAyB,SAAQ,oBAAoB;IAC7D,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,oHAAoH;IACpH,QAAQ,CAAC,OAAO,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACpD,qDAAqD;IACrD,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,aAAa,EAAE,GAAG,YAAY,CAAC;IACzD,oFAAoF;IACpF,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,GAAG,gBAAgB,CAAC;IAC3C,QAAQ,CAAC,MAAM,CAAC,EAAE,0BAA0B,CAAC,qBAAqB,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9E,wGAAwG;IACxG,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,mBAAmB,CAAC,qBAAqB,CAAC,KAAK,IAAI,CAAC;IACpF,+FAA+F;IAC/F,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,KAAK,IAAI,CAAC;CAC1D;AAED,wCAAwC;AACxC,UAAU,0BAA2B,SAAQ,wBAAwB;IACnE,sGAAsG;IACtG,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC;CAChC;AAED,mFAAmF;AACnF,UAAU,4BAA6B,SAAQ,wBAAwB;IACrE,QAAQ,CAAC,aAAa,EAAE,CAAC,OAAO,EAAE;QAAE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAA;KAAE,KAAK,OAAO,CAAC,qBAAqB,CAAC,CAAC;IAC5G,QAAQ,CAAC,aAAa,CAAC,EAAE,KAAK,CAAC;CAChC;AAED,MAAM,MAAM,kBAAkB,GAAG,0BAA0B,GAAG,4BAA4B,CAAC;AAK3F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,aAAa,CAAC,EAC5B,aAAa,EACb,aAAa,EACb,QAA2B,EAC3B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,MAAM,EACN,OAAO,EACP,SAAS,EACT,GAAG,MAAM,EACV,EAAE,kBAAkB,+BAuCpB"}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LiveAssistant = LiveAssistant;
|
|
4
|
+
const jsx_runtime_1 = require("react/jsx-runtime");
|
|
5
|
+
const react_1 = require("react");
|
|
6
|
+
const core_1 = require("@live-assistant/core");
|
|
7
|
+
const audio_1 = require("@live-assistant/audio");
|
|
8
|
+
const gemini_1 = require("@live-assistant/gemini");
|
|
9
|
+
const react_2 = require("@live-assistant/react");
|
|
10
|
+
const widget_1 = require("@live-assistant/widget");
|
|
11
|
+
const DEFAULT_LANGUAGE = 'en-US';
|
|
12
|
+
const JSON_TYPE = 'application/json';
|
|
13
|
+
/**
|
|
14
|
+
* The whole integration: import it, put it in your tree, and there is a voice
|
|
15
|
+
* assistant on the screen.
|
|
16
|
+
*
|
|
17
|
+
* ```tsx
|
|
18
|
+
* <LiveAssistant tokenEndpoint="https://api.example.com/assistant/token" />
|
|
19
|
+
* ```
|
|
20
|
+
*
|
|
21
|
+
* @remarks
|
|
22
|
+
* - **One required value, because it is the only one we cannot know**: the
|
|
23
|
+
* route on your server that mints a short-lived token. Everything else — the
|
|
24
|
+
* Gemini session, the microphone, the player, the tool registry, the
|
|
25
|
+
* controller, the provider and the widget — is built here, the same way in
|
|
26
|
+
* every app, which is what makes it ours to write rather than yours.
|
|
27
|
+
* - **The controller is built once**, in a `useState` initialiser. It owns a
|
|
28
|
+
* socket and two devices; rebuilding it on a render would start a session per
|
|
29
|
+
* render. The session is stopped when this unmounts.
|
|
30
|
+
* - **`headers` and `language` are read at connection time, not at mount.** An
|
|
31
|
+
* app whose `Authorization` header is refreshed mid-session would otherwise
|
|
32
|
+
* reconnect with the token it had when the component first rendered — which
|
|
33
|
+
* is exactly when a long conversation hands over to a new socket.
|
|
34
|
+
* - **A non-2xx from your endpoint is thrown on purpose.** It reaches the app
|
|
35
|
+
* as `connection_refused` with the response in `failure.cause`, rather than
|
|
36
|
+
* as a session that fails later for no stated reason.
|
|
37
|
+
* - **It does not own your navigation or your screens.** The page pack reads
|
|
38
|
+
* the live DOM, so on the web it already works; a native app passes its own
|
|
39
|
+
* tools, or `page: { router }`.
|
|
40
|
+
* - **Outgrowing it costs nothing**: build `AssistantController` yourself and
|
|
41
|
+
* render `AssistantProvider` + `AssistantWidget`, which is all this does.
|
|
42
|
+
*/
|
|
43
|
+
function LiveAssistant({ tokenEndpoint, getConnection, language = DEFAULT_LANGUAGE, headers, tools, page, timing, onReady, onFailure, ...widget }) {
|
|
44
|
+
// Read at connection time rather than captured at mount: see the doc block.
|
|
45
|
+
const latest = (0, react_1.useRef)({ tokenEndpoint, getConnection, language, headers });
|
|
46
|
+
latest.current = { tokenEndpoint, getConnection, language, headers };
|
|
47
|
+
const [controller] = (0, react_1.useState)(() => new core_1.AssistantController({
|
|
48
|
+
session: new gemini_1.GeminiLiveSession(),
|
|
49
|
+
microphone: new audio_1.Microphone(),
|
|
50
|
+
player: new audio_1.PcmPlayer(),
|
|
51
|
+
tools: tools instanceof core_1.ToolRegistry ? tools : new core_1.ToolRegistry(tools ?? []),
|
|
52
|
+
...(page === undefined ? {} : { page }),
|
|
53
|
+
...(timing === undefined ? {} : { timing }),
|
|
54
|
+
getConnection: async ({ resumptionHandle }) => {
|
|
55
|
+
const current = latest.current;
|
|
56
|
+
if (current.getConnection !== undefined)
|
|
57
|
+
return current.getConnection({ resumptionHandle });
|
|
58
|
+
const response = await fetch(current.tokenEndpoint, {
|
|
59
|
+
method: 'POST',
|
|
60
|
+
headers: { 'content-type': JSON_TYPE, ...current.headers },
|
|
61
|
+
body: JSON.stringify({ resumptionHandle, languageCode: current.language }),
|
|
62
|
+
});
|
|
63
|
+
if (!response.ok)
|
|
64
|
+
throw new Error(`the token endpoint answered ${response.status}`);
|
|
65
|
+
return (await response.json());
|
|
66
|
+
},
|
|
67
|
+
}));
|
|
68
|
+
(0, react_1.useEffect)(() => {
|
|
69
|
+
onReady?.(controller);
|
|
70
|
+
return () => void controller.stop();
|
|
71
|
+
}, [controller, onReady]);
|
|
72
|
+
return ((0, jsx_runtime_1.jsxs)(react_2.AssistantProvider, { controller: controller, children: [(0, jsx_runtime_1.jsx)(FailureReporter, { onFailure: onFailure }), (0, jsx_runtime_1.jsx)(widget_1.AssistantWidget, { ...widget })] }));
|
|
73
|
+
}
|
|
74
|
+
const selectError = (state) => state.error;
|
|
75
|
+
/** Reports failures to the app without re-rendering the widget for them. */
|
|
76
|
+
function FailureReporter({ onFailure }) {
|
|
77
|
+
const error = (0, react_2.useAssistantState)(selectError);
|
|
78
|
+
(0, react_1.useEffect)(() => {
|
|
79
|
+
if (error !== null)
|
|
80
|
+
onFailure?.(error);
|
|
81
|
+
}, [error, onFailure]);
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
//# sourceMappingURL=live-assistant.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"live-assistant.js","sourceRoot":"","sources":["../src/live-assistant.tsx"],"names":[],"mappings":";;AA2EA,sCAkDC;;AA7HD,iCAAoD;AACpD,+CAAyE;AAEzE,iDAA8D;AAC9D,mDAA2D;AAE3D,iDAA6E;AAC7E,mDAAyD;AAmCzD,MAAM,gBAAgB,GAAG,OAAO,CAAC;AACjC,MAAM,SAAS,GAAG,kBAAkB,CAAC;AAErC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,SAAgB,aAAa,CAAC,EAC5B,aAAa,EACb,aAAa,EACb,QAAQ,GAAG,gBAAgB,EAC3B,OAAO,EACP,KAAK,EACL,IAAI,EACJ,MAAM,EACN,OAAO,EACP,SAAS,EACT,GAAG,MAAM,EACU;IACnB,4EAA4E;IAC5E,MAAM,MAAM,GAAG,IAAA,cAAM,EAAC,EAAE,aAAa,EAAE,aAAa,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;IAC3E,MAAM,CAAC,OAAO,GAAG,EAAE,aAAa,EAAE,aAAa,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;IAErE,MAAM,CAAC,UAAU,CAAC,GAAG,IAAA,gBAAQ,EAC3B,GAAG,EAAE,CACH,IAAI,0BAAmB,CAAwB;QAC7C,OAAO,EAAE,IAAI,0BAAiB,EAAE;QAChC,UAAU,EAAE,IAAI,kBAAU,EAAE;QAC5B,MAAM,EAAE,IAAI,iBAAS,EAAE;QACvB,KAAK,EAAE,KAAK,YAAY,mBAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,mBAAY,CAAC,KAAK,IAAI,EAAE,CAAC;QAC5E,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACvC,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3C,aAAa,EAAE,KAAK,EAAE,EAAE,gBAAgB,EAAE,EAAE,EAAE;YAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YAC/B,IAAI,OAAO,CAAC,aAAa,KAAK,SAAS;gBAAE,OAAO,OAAO,CAAC,aAAa,CAAC,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAC5F,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,CAAC,aAAuB,EAAE;gBAC5D,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE;gBAC1D,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,gBAAgB,EAAE,YAAY,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;aAC3E,CAAC,CAAC;YACH,IAAI,CAAC,QAAQ,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACpF,OAAO,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA0B,CAAC;QAC1D,CAAC;KACF,CAAC,CACL,CAAC;IAEF,IAAA,iBAAS,EAAC,GAAG,EAAE;QACb,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;QACtB,OAAO,GAAG,EAAE,CAAC,KAAK,UAAU,CAAC,IAAI,EAAE,CAAC;IACtC,CAAC,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;IAE1B,OAAO,CACL,wBAAC,yBAAiB,IAAC,UAAU,EAAE,UAAU,aACvC,uBAAC,eAAe,IAAC,SAAS,EAAE,SAAS,GAAI,EACzC,uBAAC,wBAAe,OAAK,MAAM,GAAI,IACb,CACrB,CAAC;AACJ,CAAC;AAED,MAAM,WAAW,GAAG,CAAC,KAAkD,EAA2B,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;AAEjH,4EAA4E;AAC5E,SAAS,eAAe,CAAC,EAAE,SAAS,EAAgE;IAClG,MAAM,KAAK,GAAG,IAAA,yBAAiB,EAAC,WAAW,CAAC,CAAC;IAC7C,IAAA,iBAAS,EAAC,GAAG,EAAE;QACb,IAAI,KAAK,KAAK,IAAI;YAAE,SAAS,EAAE,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;IACvB,OAAO,IAAI,CAAC;AACd,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@live-assistant/react-native",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "One install for a voice assistant in a React Native or Expo app: the session, a Gemini Live connection, microphone and playback, React bindings and a ready-made widget, all re-exported from one place.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Recep Tayyip Ekşi",
|
|
@@ -28,6 +28,7 @@
|
|
|
28
28
|
"src",
|
|
29
29
|
"README.md",
|
|
30
30
|
"LICENSE",
|
|
31
|
+
"app.plugin.js",
|
|
31
32
|
"!src/**/__tests__",
|
|
32
33
|
"!src/**/__fixtures__"
|
|
33
34
|
],
|
|
@@ -40,11 +41,11 @@
|
|
|
40
41
|
"access": "public"
|
|
41
42
|
},
|
|
42
43
|
"dependencies": {
|
|
43
|
-
"@live-assistant/audio": "0.
|
|
44
|
-
"@live-assistant/core": "0.
|
|
45
|
-
"@live-assistant/gemini": "0.
|
|
46
|
-
"@live-assistant/react": "0.
|
|
47
|
-
"@live-assistant/widget": "0.
|
|
44
|
+
"@live-assistant/audio": "0.3.0",
|
|
45
|
+
"@live-assistant/core": "0.3.0",
|
|
46
|
+
"@live-assistant/gemini": "0.3.0",
|
|
47
|
+
"@live-assistant/react": "0.3.0",
|
|
48
|
+
"@live-assistant/widget": "0.3.0"
|
|
48
49
|
},
|
|
49
50
|
"peerDependencies": {
|
|
50
51
|
"react": ">=18",
|
package/src/index.ts
CHANGED
|
@@ -9,10 +9,13 @@
|
|
|
9
9
|
* should be able to leave out. None of those reasons apply to the ordinary
|
|
10
10
|
* case — an app that wants the assistant — and making that case install five
|
|
11
11
|
* packages and keep five versions in step is friction with nothing behind it.
|
|
12
|
-
* - **It adds
|
|
13
|
-
* so reading the source of
|
|
14
|
-
* are holding, and an app that
|
|
15
|
-
* pieces directly without changing
|
|
12
|
+
* - **It adds exactly one thing of its own: `<LiveAssistant>`.** Everything
|
|
13
|
+
* else is re-exported unchanged, so reading the source of
|
|
14
|
+
* `@live-assistant/core` still explains what you are holding, and an app that
|
|
15
|
+
* outgrows this package can depend on the pieces directly without changing an
|
|
16
|
+
* import. The one addition exists because this is the only package that can
|
|
17
|
+
* see the session, the audio and the widget at once — and because the forty
|
|
18
|
+
* lines it replaces were identical in every app that would ever write them.
|
|
16
19
|
* - **The token server is deliberately absent.** It mints credentials with an
|
|
17
20
|
* API key, which belongs on a server and never in an app bundle. It is
|
|
18
21
|
* installed on its own, where it runs.
|
|
@@ -22,3 +25,5 @@ export * from '@live-assistant/gemini';
|
|
|
22
25
|
export * from '@live-assistant/audio';
|
|
23
26
|
export * from '@live-assistant/react';
|
|
24
27
|
export * from '@live-assistant/widget';
|
|
28
|
+
export { LiveAssistant } from './live-assistant';
|
|
29
|
+
export type { LiveAssistantProps } from './live-assistant';
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { useEffect, useRef, useState } from 'react';
|
|
2
|
+
import { AssistantController, ToolRegistry } from '@live-assistant/core';
|
|
3
|
+
import type { AssistantControllerOptions, AssistantFailure, AssistantTool, PageToolsOptions } from '@live-assistant/core';
|
|
4
|
+
import { Microphone, PcmPlayer } from '@live-assistant/audio';
|
|
5
|
+
import { GeminiLiveSession } from '@live-assistant/gemini';
|
|
6
|
+
import type { GeminiLiveCredentials } from '@live-assistant/gemini';
|
|
7
|
+
import { AssistantProvider, useAssistantState } from '@live-assistant/react';
|
|
8
|
+
import { AssistantWidget } from '@live-assistant/widget';
|
|
9
|
+
import type { AssistantWidgetProps } from '@live-assistant/widget';
|
|
10
|
+
|
|
11
|
+
/** Shared by both ways of connecting; see `LiveAssistantProps`. */
|
|
12
|
+
interface LiveAssistantCommonProps extends AssistantWidgetProps {
|
|
13
|
+
/** Sent to your token endpoint as `languageCode`. Default `'en-US'`. */
|
|
14
|
+
readonly language?: string;
|
|
15
|
+
/** Added to the token request — this is where an app's own `Authorization` goes. Read fresh on every connection. */
|
|
16
|
+
readonly headers?: Readonly<Record<string, string>>;
|
|
17
|
+
/** Your app's own tools, on top of the page pack. */
|
|
18
|
+
readonly tools?: readonly AssistantTool[] | ToolRegistry;
|
|
19
|
+
/** Reading and driving the page. On by default in a browser; `false` removes it. */
|
|
20
|
+
readonly page?: boolean | PageToolsOptions;
|
|
21
|
+
readonly timing?: AssistantControllerOptions<GeminiLiveCredentials>['timing'];
|
|
22
|
+
/** Called with the controller once, for an app that wants to start or stop a session from elsewhere. */
|
|
23
|
+
readonly onReady?: (controller: AssistantController<GeminiLiveCredentials>) => void;
|
|
24
|
+
/** Called whenever a session fails. The widget already tells the user; this is for logging. */
|
|
25
|
+
readonly onFailure?: (failure: AssistantFailure) => void;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** The endpoint form: the usual one. */
|
|
29
|
+
interface LiveAssistantEndpointProps extends LiveAssistantCommonProps {
|
|
30
|
+
/** Your server's route that mints a short-lived token. The one value this component cannot invent. */
|
|
31
|
+
readonly tokenEndpoint: string;
|
|
32
|
+
readonly getConnection?: never;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The escape hatch: an app that already has its own client for the same thing. */
|
|
36
|
+
interface LiveAssistantConnectionProps extends LiveAssistantCommonProps {
|
|
37
|
+
readonly getConnection: (request: { readonly resumptionHandle?: string }) => Promise<GeminiLiveCredentials>;
|
|
38
|
+
readonly tokenEndpoint?: never;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export type LiveAssistantProps = LiveAssistantEndpointProps | LiveAssistantConnectionProps;
|
|
42
|
+
|
|
43
|
+
const DEFAULT_LANGUAGE = 'en-US';
|
|
44
|
+
const JSON_TYPE = 'application/json';
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The whole integration: import it, put it in your tree, and there is a voice
|
|
48
|
+
* assistant on the screen.
|
|
49
|
+
*
|
|
50
|
+
* ```tsx
|
|
51
|
+
* <LiveAssistant tokenEndpoint="https://api.example.com/assistant/token" />
|
|
52
|
+
* ```
|
|
53
|
+
*
|
|
54
|
+
* @remarks
|
|
55
|
+
* - **One required value, because it is the only one we cannot know**: the
|
|
56
|
+
* route on your server that mints a short-lived token. Everything else — the
|
|
57
|
+
* Gemini session, the microphone, the player, the tool registry, the
|
|
58
|
+
* controller, the provider and the widget — is built here, the same way in
|
|
59
|
+
* every app, which is what makes it ours to write rather than yours.
|
|
60
|
+
* - **The controller is built once**, in a `useState` initialiser. It owns a
|
|
61
|
+
* socket and two devices; rebuilding it on a render would start a session per
|
|
62
|
+
* render. The session is stopped when this unmounts.
|
|
63
|
+
* - **`headers` and `language` are read at connection time, not at mount.** An
|
|
64
|
+
* app whose `Authorization` header is refreshed mid-session would otherwise
|
|
65
|
+
* reconnect with the token it had when the component first rendered — which
|
|
66
|
+
* is exactly when a long conversation hands over to a new socket.
|
|
67
|
+
* - **A non-2xx from your endpoint is thrown on purpose.** It reaches the app
|
|
68
|
+
* as `connection_refused` with the response in `failure.cause`, rather than
|
|
69
|
+
* as a session that fails later for no stated reason.
|
|
70
|
+
* - **It does not own your navigation or your screens.** The page pack reads
|
|
71
|
+
* the live DOM, so on the web it already works; a native app passes its own
|
|
72
|
+
* tools, or `page: { router }`.
|
|
73
|
+
* - **Outgrowing it costs nothing**: build `AssistantController` yourself and
|
|
74
|
+
* render `AssistantProvider` + `AssistantWidget`, which is all this does.
|
|
75
|
+
*/
|
|
76
|
+
export function LiveAssistant({
|
|
77
|
+
tokenEndpoint,
|
|
78
|
+
getConnection,
|
|
79
|
+
language = DEFAULT_LANGUAGE,
|
|
80
|
+
headers,
|
|
81
|
+
tools,
|
|
82
|
+
page,
|
|
83
|
+
timing,
|
|
84
|
+
onReady,
|
|
85
|
+
onFailure,
|
|
86
|
+
...widget
|
|
87
|
+
}: LiveAssistantProps) {
|
|
88
|
+
// Read at connection time rather than captured at mount: see the doc block.
|
|
89
|
+
const latest = useRef({ tokenEndpoint, getConnection, language, headers });
|
|
90
|
+
latest.current = { tokenEndpoint, getConnection, language, headers };
|
|
91
|
+
|
|
92
|
+
const [controller] = useState(
|
|
93
|
+
() =>
|
|
94
|
+
new AssistantController<GeminiLiveCredentials>({
|
|
95
|
+
session: new GeminiLiveSession(),
|
|
96
|
+
microphone: new Microphone(),
|
|
97
|
+
player: new PcmPlayer(),
|
|
98
|
+
tools: tools instanceof ToolRegistry ? tools : new ToolRegistry(tools ?? []),
|
|
99
|
+
...(page === undefined ? {} : { page }),
|
|
100
|
+
...(timing === undefined ? {} : { timing }),
|
|
101
|
+
getConnection: async ({ resumptionHandle }) => {
|
|
102
|
+
const current = latest.current;
|
|
103
|
+
if (current.getConnection !== undefined) return current.getConnection({ resumptionHandle });
|
|
104
|
+
const response = await fetch(current.tokenEndpoint as string, {
|
|
105
|
+
method: 'POST',
|
|
106
|
+
headers: { 'content-type': JSON_TYPE, ...current.headers },
|
|
107
|
+
body: JSON.stringify({ resumptionHandle, languageCode: current.language }),
|
|
108
|
+
});
|
|
109
|
+
if (!response.ok) throw new Error(`the token endpoint answered ${response.status}`);
|
|
110
|
+
return (await response.json()) as GeminiLiveCredentials;
|
|
111
|
+
},
|
|
112
|
+
}),
|
|
113
|
+
);
|
|
114
|
+
|
|
115
|
+
useEffect(() => {
|
|
116
|
+
onReady?.(controller);
|
|
117
|
+
return () => void controller.stop();
|
|
118
|
+
}, [controller, onReady]);
|
|
119
|
+
|
|
120
|
+
return (
|
|
121
|
+
<AssistantProvider controller={controller}>
|
|
122
|
+
<FailureReporter onFailure={onFailure} />
|
|
123
|
+
<AssistantWidget {...widget} />
|
|
124
|
+
</AssistantProvider>
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const selectError = (state: { readonly error: AssistantFailure | null }): AssistantFailure | null => state.error;
|
|
129
|
+
|
|
130
|
+
/** Reports failures to the app without re-rendering the widget for them. */
|
|
131
|
+
function FailureReporter({ onFailure }: { readonly onFailure?: (failure: AssistantFailure) => void }) {
|
|
132
|
+
const error = useAssistantState(selectError);
|
|
133
|
+
useEffect(() => {
|
|
134
|
+
if (error !== null) onFailure?.(error);
|
|
135
|
+
}, [error, onFailure]);
|
|
136
|
+
return null;
|
|
137
|
+
}
|