@alia.onl/sdk 13.1.0 → 13.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/README.md +31 -0
- package/package.json +1 -1
- package/src/components/AliaChatContent.tsx +53 -100
- package/src/components/AliaChatScreen.tsx +27 -24
- package/src/components/AliaChatSheet.tsx +15 -4
- package/src/components/AliaComposer.tsx +336 -0
- package/src/components/ComposerDropZone.tsx +235 -0
- package/src/hooks/useAttachmentIntake.ts +401 -0
- package/src/hooks/useFocusWhenPresented.ts +41 -0
- package/src/hooks/usePendingQuestions.ts +19 -0
- package/src/hooks/useVoiceRoom.ts +81 -16
- package/src/index.ts +10 -0
- package/src/lib/attachment-intake.ts +154 -0
- package/src/lib/attachment-tiles.ts +74 -0
- package/src/lib/labels.tsx +47 -1
- package/src/lib/speech-messages.ts +34 -2
- package/src/lib/speech-synthesis.ts +99 -8
- package/src/tools.ts +1 -0
- package/src/types.ts +14 -0
- package/src/voice.ts +2 -0
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
import React from 'react';
|
|
2
|
+
import {
|
|
3
|
+
classifyIntake,
|
|
4
|
+
readFraction,
|
|
5
|
+
type IntakeKind,
|
|
6
|
+
type IntakeRefusal,
|
|
7
|
+
} from '../lib/attachment-intake';
|
|
8
|
+
import type { AliaAttachment } from '../types';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The composer's own file intake: the queue between "the user handed us a
|
|
12
|
+
* file" and "the attachment strip has it".
|
|
13
|
+
*
|
|
14
|
+
* ## There is no upload
|
|
15
|
+
*
|
|
16
|
+
* This is the first thing to know, and it decides everything below. Nothing in
|
|
17
|
+
* Alia uploads an attachment. A picked file becomes a `uri` and sits in the
|
|
18
|
+
* store until the message is sent, at which point the host's send
|
|
19
|
+
* inlines the images into the request body as `data:` URLs and drops the
|
|
20
|
+
* documents on the floor. There is no endpoint, no `FormData`, no XHR — so
|
|
21
|
+
* there are no bytes in flight to measure and nothing an abort could stop.
|
|
22
|
+
*
|
|
23
|
+
* What there IS, on web, is a read. A `File` that arrives by paste or by drop
|
|
24
|
+
* is a handle, not bytes, and it has to be read into a `data:` URL before it
|
|
25
|
+
* can be sent — a real transfer of real bytes, of a size the user chose, that
|
|
26
|
+
* takes real time for anything above a few megabytes. `FileReader` reports
|
|
27
|
+
* that read through `progress` events carrying `lengthComputable`, `loaded`
|
|
28
|
+
* and `total`, and it can be stopped with `abort()`. So the progress this
|
|
29
|
+
* queue shows is measured and the cancel button ends something: they are the
|
|
30
|
+
* read, honestly labelled, and not a stand-in for an upload that does not
|
|
31
|
+
* exist. When the browser declines to make the read measurable the tile says
|
|
32
|
+
* so by showing indeterminate activity — see `readFraction`.
|
|
33
|
+
*
|
|
34
|
+
* ## Why the queue is separate from the attachment list
|
|
35
|
+
*
|
|
36
|
+
* A file that is still being read is not an attachment yet: it has no `uri`,
|
|
37
|
+
* so `buildMessageContent` would silently skip it, and the strip would show a
|
|
38
|
+
* tile that contributes nothing to the message. It also cannot be REPAIRED in
|
|
39
|
+
* place — `PromptInput` only forwards an `onUpdateAttachment` when its consumer
|
|
40
|
+
* supplies one, and the chat page supplies `attachments`, `onAddAttachment`
|
|
41
|
+
* and `onRemoveAttachment` and no updater at all, so every `updateAttachment`
|
|
42
|
+
* from inside the composer lands in internal state that the controlled list
|
|
43
|
+
* shadows and nothing ever renders. (That is why the old paste path reached
|
|
44
|
+
* around the composer and called `useStore.getState().updateAttachment`
|
|
45
|
+
* directly.) Holding the in-flight files here instead means the strip's
|
|
46
|
+
* pending rows work the same whether the consumer wired an updater or not, and
|
|
47
|
+
* an attachment is only ever handed over once it has bytes behind it.
|
|
48
|
+
*/
|
|
49
|
+
|
|
50
|
+
/** A file the composer is working on, has failed to read, or will not take. */
|
|
51
|
+
export interface IntakeItem {
|
|
52
|
+
id: string;
|
|
53
|
+
name: string;
|
|
54
|
+
size: number;
|
|
55
|
+
mimeType: string;
|
|
56
|
+
kind: IntakeKind;
|
|
57
|
+
/**
|
|
58
|
+
* `reading` while the bytes move, `failed` once they did not, `refused` for
|
|
59
|
+
* a file that never started — too big, or a dropped folder.
|
|
60
|
+
*
|
|
61
|
+
* A refusal is a queue entry rather than a toast for two reasons. It stays
|
|
62
|
+
* put: a drop of six files where one was too large is a sentence the user
|
|
63
|
+
* needs beside the five that worked, not one that slides away after four
|
|
64
|
+
* seconds. And it keeps the composer free of a notification dependency —
|
|
65
|
+
* `@oxy.so/bloom/toast` is a module the composer's own test suites do not
|
|
66
|
+
* stub, and a component that cannot be mounted in a test is a component
|
|
67
|
+
* whose behaviour nothing pins.
|
|
68
|
+
*/
|
|
69
|
+
status: 'reading' | 'failed' | 'refused';
|
|
70
|
+
/**
|
|
71
|
+
* How far the read has got, 0–1 — or `null` for "no measurable progress",
|
|
72
|
+
* which the tile must render as indeterminate activity rather than as a
|
|
73
|
+
* number. Never a timer, never a guess.
|
|
74
|
+
*/
|
|
75
|
+
fraction: number | null;
|
|
76
|
+
/** Set only on a `refused` row: which sentence to show. */
|
|
77
|
+
refusal?: IntakeRefusal;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** A started read, and the one thing a caller can do to it. */
|
|
81
|
+
export interface ReadHandle {
|
|
82
|
+
abort: () => void;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface ReadHandlers {
|
|
86
|
+
onProgress: (fraction: number | null) => void;
|
|
87
|
+
onDone: (dataUrl: string) => void;
|
|
88
|
+
onFailed: () => void;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Begins a read and hands back the way to stop it. Injectable because
|
|
93
|
+
* `FileReader` is a browser API and the tests run in node — and because a test
|
|
94
|
+
* that cannot drive a read step by step cannot pin what cancel and retry do.
|
|
95
|
+
*/
|
|
96
|
+
export type StartRead = (file: Blob, handlers: ReadHandlers) => ReadHandle;
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* The real reader: `FileReader`, the app's ONLY genuine cancellation point in
|
|
100
|
+
* the whole attachment path.
|
|
101
|
+
*/
|
|
102
|
+
const startFileRead: StartRead = (file, handlers) => {
|
|
103
|
+
const reader = new FileReader();
|
|
104
|
+
reader.onprogress = (event) => handlers.onProgress(readFraction(event));
|
|
105
|
+
reader.onload = () => {
|
|
106
|
+
const result = reader.result;
|
|
107
|
+
// A `readAsDataURL` result is always a string when it succeeded; anything
|
|
108
|
+
// else means the read did not produce what the message needs, and a
|
|
109
|
+
// failure the user can retry is the truthful reading of that.
|
|
110
|
+
if (typeof result === 'string') handlers.onDone(result);
|
|
111
|
+
else handlers.onFailed();
|
|
112
|
+
};
|
|
113
|
+
reader.onerror = () => handlers.onFailed();
|
|
114
|
+
reader.readAsDataURL(file);
|
|
115
|
+
return { abort: () => reader.abort() };
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
export interface AttachmentIntake {
|
|
119
|
+
/** In-flight and failed files, in the order they arrived. */
|
|
120
|
+
items: IntakeItem[];
|
|
121
|
+
/** Take a batch of files from a paste or a drop. */
|
|
122
|
+
accept: (files: readonly File[]) => void;
|
|
123
|
+
/** Stop a read and drop the file. */
|
|
124
|
+
cancel: (id: string) => void;
|
|
125
|
+
/** Read a failed file again, from the file itself. */
|
|
126
|
+
retry: (id: string) => void;
|
|
127
|
+
/** Give up on a failed or refused file. */
|
|
128
|
+
dismiss: (id: string) => void;
|
|
129
|
+
/** True while anything is still being read — nothing may be sent yet. */
|
|
130
|
+
isBusy: boolean;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export interface AttachmentIntakeOptions {
|
|
134
|
+
addAttachment: (attachment: AliaAttachment) => void;
|
|
135
|
+
/**
|
|
136
|
+
* What the host's send can carry. Omitted, both; a file of a kind left out is refused
|
|
137
|
+
* (`unsupported`) instead of landing as a tile the send would drop.
|
|
138
|
+
*/
|
|
139
|
+
kinds?: readonly IntakeKind[];
|
|
140
|
+
/** Swapped out by the tests; the real one is `startFileRead`. */
|
|
141
|
+
startRead?: StartRead;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const BOTH_KINDS: readonly IntakeKind[] = ['image', 'document'];
|
|
145
|
+
|
|
146
|
+
/** Ids that cannot collide, without a `Math.random()` anywhere near progress. */
|
|
147
|
+
let sequence = 0;
|
|
148
|
+
const nextId = (prefix: string) => `${prefix}-${Date.now()}-${++sequence}`;
|
|
149
|
+
|
|
150
|
+
export function useAttachmentIntake({
|
|
151
|
+
addAttachment,
|
|
152
|
+
kinds = BOTH_KINDS,
|
|
153
|
+
startRead = startFileRead,
|
|
154
|
+
}: AttachmentIntakeOptions): AttachmentIntake {
|
|
155
|
+
const [items, setItems] = React.useState<IntakeItem[]>([]);
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* The live reads and the files behind them, by item id.
|
|
159
|
+
*
|
|
160
|
+
* Refs rather than state, and deliberately so: a read handle is not
|
|
161
|
+
* something the strip draws, and re-rendering the composer every time a
|
|
162
|
+
* reader is registered would be a render per file per keystroke-free tick.
|
|
163
|
+
* More importantly `cancel` must reach the reader that is running NOW — a
|
|
164
|
+
* handle captured in a closure over state would be one render stale, which
|
|
165
|
+
* for a cancel button means aborting the read before last.
|
|
166
|
+
*
|
|
167
|
+
* The file is kept alongside for exactly one reason: retry has to re-read
|
|
168
|
+
* something, and the `File` is the only thing that still holds the bytes.
|
|
169
|
+
* Both maps are cleared the moment an item leaves the queue, which is what
|
|
170
|
+
* makes "release the reading resources" true rather than aspirational.
|
|
171
|
+
*/
|
|
172
|
+
const reads = React.useRef(new Map<string, ReadHandle>());
|
|
173
|
+
const sources = React.useRef(new Map<string, File>());
|
|
174
|
+
|
|
175
|
+
const forget = React.useCallback((id: string) => {
|
|
176
|
+
reads.current.get(id)?.abort();
|
|
177
|
+
reads.current.delete(id);
|
|
178
|
+
sources.current.delete(id);
|
|
179
|
+
}, []);
|
|
180
|
+
|
|
181
|
+
const begin = React.useCallback(
|
|
182
|
+
(id: string, file: File, kind: IntakeKind) => {
|
|
183
|
+
/**
|
|
184
|
+
* Whether this read is still the one we are waiting on.
|
|
185
|
+
*
|
|
186
|
+
* `abort()` is a request, not a guarantee: a `FileReader` may still
|
|
187
|
+
* deliver a queued event afterwards, and some engines fire `load` for a
|
|
188
|
+
* read that had already completed in the background when the abort
|
|
189
|
+
* arrived. Without this check a cancelled file lands in the attachment
|
|
190
|
+
* list a beat after the user removed it — the row comes BACK — which is
|
|
191
|
+
* the failure mode that makes a cancel button worse than none.
|
|
192
|
+
*
|
|
193
|
+
* The map is the record of what is live, so asking it is also what makes
|
|
194
|
+
* "cancel" and "unmount" mean the same thing here.
|
|
195
|
+
*/
|
|
196
|
+
const live = () => reads.current.has(id);
|
|
197
|
+
|
|
198
|
+
/*
|
|
199
|
+
* Registered BEFORE the read starts, then swapped for the real handle.
|
|
200
|
+
* A cached file can be read synchronously — the browser has the bytes
|
|
201
|
+
* already — and `onDone` would then run inside `startRead`, before there
|
|
202
|
+
* was anything in the map to say this read was live. The placeholder is
|
|
203
|
+
* what stops that completion being mistaken for a stale one and dropped.
|
|
204
|
+
*/
|
|
205
|
+
const placeholder: ReadHandle = { abort: () => {} };
|
|
206
|
+
reads.current.set(id, placeholder);
|
|
207
|
+
|
|
208
|
+
const handle = startRead(file, {
|
|
209
|
+
onProgress: (fraction) => {
|
|
210
|
+
if (!live()) return;
|
|
211
|
+
setItems((prev) =>
|
|
212
|
+
prev.map((item) =>
|
|
213
|
+
item.id === id && item.status === 'reading' ? { ...item, fraction } : item,
|
|
214
|
+
),
|
|
215
|
+
);
|
|
216
|
+
},
|
|
217
|
+
onDone: (dataUrl) => {
|
|
218
|
+
if (!live()) return;
|
|
219
|
+
// The read is over before the attachment exists, so the handle and
|
|
220
|
+
// the file go first: a completed reader kept in the map is a `File`
|
|
221
|
+
// kept alive, which for a 20 MiB drop is 20 MiB held for the life of
|
|
222
|
+
// the composer.
|
|
223
|
+
reads.current.delete(id);
|
|
224
|
+
sources.current.delete(id);
|
|
225
|
+
setItems((prev) => prev.filter((item) => item.id !== id));
|
|
226
|
+
addAttachment({
|
|
227
|
+
id,
|
|
228
|
+
uri: dataUrl,
|
|
229
|
+
type: kind,
|
|
230
|
+
name: file.name,
|
|
231
|
+
size: file.size,
|
|
232
|
+
mimeType: file.type || 'application/octet-stream',
|
|
233
|
+
});
|
|
234
|
+
},
|
|
235
|
+
onFailed: () => {
|
|
236
|
+
if (!live()) return;
|
|
237
|
+
// The reader is spent; the FILE is not, and retry needs it. This is
|
|
238
|
+
// the one path that keeps a source around after its read ended.
|
|
239
|
+
reads.current.delete(id);
|
|
240
|
+
setItems((prev) =>
|
|
241
|
+
prev.map((item) =>
|
|
242
|
+
item.id === id ? { ...item, status: 'failed', fraction: null } : item,
|
|
243
|
+
),
|
|
244
|
+
);
|
|
245
|
+
},
|
|
246
|
+
});
|
|
247
|
+
// Only if the placeholder is still there. If the read already finished
|
|
248
|
+
// or failed inside the call above, the map has been moved on and writing
|
|
249
|
+
// the handle back would leave a spent reader registered as live.
|
|
250
|
+
if (reads.current.get(id) === placeholder) reads.current.set(id, handle);
|
|
251
|
+
},
|
|
252
|
+
[addAttachment, startRead],
|
|
253
|
+
);
|
|
254
|
+
|
|
255
|
+
const accept = React.useCallback(
|
|
256
|
+
(files: readonly File[]) => {
|
|
257
|
+
const queued: IntakeItem[] = [];
|
|
258
|
+
const starts: Array<() => void> = [];
|
|
259
|
+
|
|
260
|
+
for (const file of files) {
|
|
261
|
+
const name = file.name || '';
|
|
262
|
+
const verdict = classifyIntake({
|
|
263
|
+
name,
|
|
264
|
+
mimeType: file.type || '',
|
|
265
|
+
size: file.size,
|
|
266
|
+
});
|
|
267
|
+
const id = nextId('intake');
|
|
268
|
+
const refusal = !verdict.accepted
|
|
269
|
+
? verdict.refusal
|
|
270
|
+
: kinds.includes(verdict.kind)
|
|
271
|
+
? undefined
|
|
272
|
+
: ('unsupported' as const);
|
|
273
|
+
if (refusal !== undefined) {
|
|
274
|
+
// A file that silently fails to attach is the worst of the three
|
|
275
|
+
// outcomes: the user sends the message believing the picture went
|
|
276
|
+
// with it. The row names the file, because a drop is usually a batch
|
|
277
|
+
// and "one of them didn't work" is no answer when four were dropped.
|
|
278
|
+
queued.push({
|
|
279
|
+
id,
|
|
280
|
+
name: name || 'file',
|
|
281
|
+
size: file.size,
|
|
282
|
+
mimeType: file.type || '',
|
|
283
|
+
kind: 'document',
|
|
284
|
+
status: 'refused',
|
|
285
|
+
fraction: null,
|
|
286
|
+
refusal,
|
|
287
|
+
});
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
if (!verdict.accepted) continue;
|
|
291
|
+
if (verdict.kind === 'document') {
|
|
292
|
+
/*
|
|
293
|
+
* A document never gets read. `buildMessageContent` filters the
|
|
294
|
+
* attachment list down to `type === 'image'`, so a document's bytes
|
|
295
|
+
* go nowhere and reading them would be a progress bar over work that
|
|
296
|
+
* exists only to produce a string nobody consumes. The object URL is
|
|
297
|
+
* what the web pickers produce for the same file, it is instant, and
|
|
298
|
+
* the draft store gives it back when the tile is removed.
|
|
299
|
+
*/
|
|
300
|
+
addAttachment({
|
|
301
|
+
id,
|
|
302
|
+
uri:
|
|
303
|
+
typeof URL !== 'undefined' && typeof URL.createObjectURL === 'function'
|
|
304
|
+
? URL.createObjectURL(file)
|
|
305
|
+
: '',
|
|
306
|
+
type: 'document',
|
|
307
|
+
name: name || 'file',
|
|
308
|
+
size: file.size,
|
|
309
|
+
mimeType: file.type || 'application/octet-stream',
|
|
310
|
+
});
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
sources.current.set(id, file);
|
|
315
|
+
queued.push({
|
|
316
|
+
id,
|
|
317
|
+
name: name || 'image',
|
|
318
|
+
size: file.size,
|
|
319
|
+
mimeType: file.type || 'image/*',
|
|
320
|
+
kind: verdict.kind,
|
|
321
|
+
status: 'reading',
|
|
322
|
+
// Not zero. Zero is a claim that nothing has moved yet; `null` is the
|
|
323
|
+
// truth, which is that nothing has been reported yet.
|
|
324
|
+
fraction: null,
|
|
325
|
+
});
|
|
326
|
+
starts.push(() => begin(id, file, verdict.kind));
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
if (queued.length > 0) setItems((prev) => [...prev, ...queued]);
|
|
330
|
+
// Started after the rows exist, because a synchronous reader — the fake
|
|
331
|
+
// one in the tests, and a cached file in some browsers — can complete
|
|
332
|
+
// inside `startRead`, and a completion for a row that has not been added
|
|
333
|
+
// yet would be filtered against a list it is not in and then re-added by
|
|
334
|
+
// the batch above as a ghost that never finishes.
|
|
335
|
+
for (const start of starts) start();
|
|
336
|
+
},
|
|
337
|
+
[addAttachment, begin, kinds],
|
|
338
|
+
);
|
|
339
|
+
|
|
340
|
+
const cancel = React.useCallback(
|
|
341
|
+
(id: string) => {
|
|
342
|
+
forget(id);
|
|
343
|
+
setItems((prev) => prev.filter((item) => item.id !== id));
|
|
344
|
+
},
|
|
345
|
+
[forget],
|
|
346
|
+
);
|
|
347
|
+
|
|
348
|
+
const dismiss = cancel;
|
|
349
|
+
|
|
350
|
+
const retry = React.useCallback(
|
|
351
|
+
(id: string) => {
|
|
352
|
+
// The file IS the guard. A refused row never had one — nothing was
|
|
353
|
+
// read, and reading the same 30 MB again would reach the same verdict —
|
|
354
|
+
// and a cancelled one let go of it, so neither can start a second read
|
|
355
|
+
// by pressing a button the tile does not draw for them anyway.
|
|
356
|
+
const file = sources.current.get(id);
|
|
357
|
+
if (file === undefined) return;
|
|
358
|
+
setItems((prev) =>
|
|
359
|
+
prev.map((item) =>
|
|
360
|
+
item.id === id ? { ...item, status: 'reading', fraction: null } : item,
|
|
361
|
+
),
|
|
362
|
+
);
|
|
363
|
+
// `"image"` is not an assumption: a document is handed over on arrival
|
|
364
|
+
// and never queues, so the only thing that can hold a source is a
|
|
365
|
+
// picture being read.
|
|
366
|
+
begin(id, file, 'image');
|
|
367
|
+
},
|
|
368
|
+
[begin],
|
|
369
|
+
);
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* Nothing survives the composer going away.
|
|
373
|
+
*
|
|
374
|
+
* An in-flight `FileReader` holds its `File`, and a `File` holds the bytes;
|
|
375
|
+
* a composer unmounted mid-read (navigating away, the drawer closing) would
|
|
376
|
+
* otherwise leave both running until the read finished into a `setItems` on
|
|
377
|
+
* a dead tree. Aborting is the whole of the fix and there is nowhere else to
|
|
378
|
+
* put it.
|
|
379
|
+
*/
|
|
380
|
+
React.useEffect(() => {
|
|
381
|
+
const live = reads.current;
|
|
382
|
+
const held = sources.current;
|
|
383
|
+
return () => {
|
|
384
|
+
for (const handle of live.values()) handle.abort();
|
|
385
|
+
live.clear();
|
|
386
|
+
held.clear();
|
|
387
|
+
};
|
|
388
|
+
}, []);
|
|
389
|
+
|
|
390
|
+
return React.useMemo(
|
|
391
|
+
() => ({
|
|
392
|
+
items,
|
|
393
|
+
accept,
|
|
394
|
+
cancel,
|
|
395
|
+
retry,
|
|
396
|
+
dismiss,
|
|
397
|
+
isBusy: items.some((item) => item.status === 'reading'),
|
|
398
|
+
}),
|
|
399
|
+
[items, accept, cancel, retry, dismiss],
|
|
400
|
+
);
|
|
401
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { useCallback, useRef } from 'react';
|
|
2
|
+
import {
|
|
3
|
+
runOnJS,
|
|
4
|
+
type SharedValue,
|
|
5
|
+
useAnimatedReaction,
|
|
6
|
+
useSharedValue,
|
|
7
|
+
} from 'react-native-reanimated';
|
|
8
|
+
|
|
9
|
+
/** Bloom's sheet reports `1` once fully presented; a hair under it absorbs float rounding. */
|
|
10
|
+
const PRESENTED = 0.999;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Calls `focus` once each time a sheet finishes presenting.
|
|
14
|
+
*
|
|
15
|
+
* `progress` is the `animatedProgress` of Bloom's `BottomSheet`: `0` closed, `1` fully
|
|
16
|
+
* presented, every value between while it animates or is dragged. The sheet is armed while it
|
|
17
|
+
* is closed and fires on reaching `1`, so dragging it part-way and letting go does not take the
|
|
18
|
+
* focus again; only a new `present()` does. Reduced motion settles immediately, which is the
|
|
19
|
+
* same crossing. Omitted, nothing happens — the surface is not a sheet.
|
|
20
|
+
*/
|
|
21
|
+
export function useFocusWhenPresented(
|
|
22
|
+
progress: SharedValue<number> | undefined,
|
|
23
|
+
focus: () => void,
|
|
24
|
+
): void {
|
|
25
|
+
const latest = useRef(focus);
|
|
26
|
+
latest.current = focus;
|
|
27
|
+
const run = useCallback(() => latest.current(), []);
|
|
28
|
+
const armed = useSharedValue(true);
|
|
29
|
+
useAnimatedReaction(
|
|
30
|
+
() => progress?.value ?? 0,
|
|
31
|
+
(value) => {
|
|
32
|
+
if (value <= 0) {
|
|
33
|
+
armed.value = true;
|
|
34
|
+
} else if (value >= PRESENTED && armed.value) {
|
|
35
|
+
armed.value = false;
|
|
36
|
+
runOnJS(run)();
|
|
37
|
+
}
|
|
38
|
+
},
|
|
39
|
+
[progress, run],
|
|
40
|
+
);
|
|
41
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { useCallback, useMemo, useState } from 'react';
|
|
2
|
+
import { type AskedQuestion, pendingQuestions } from '../lib/questions';
|
|
3
|
+
import type { ChatMessage } from '../types';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* What Alia asked in her last turn and the person has not answered: asked above the composer,
|
|
7
|
+
* not in the thread. Any later message, typed or sent from the card, is the answer; `dismiss`
|
|
8
|
+
* puts the card away until she asks again.
|
|
9
|
+
*/
|
|
10
|
+
export function usePendingQuestions(
|
|
11
|
+
messages: readonly Pick<ChatMessage, 'id' | 'role' | 'toolInvocations'>[],
|
|
12
|
+
isStreaming: boolean,
|
|
13
|
+
): { questions: AskedQuestion[]; dismiss: () => void } {
|
|
14
|
+
const asked = useMemo(() => pendingQuestions(messages, isStreaming), [messages, isStreaming]);
|
|
15
|
+
const lastMessageId = messages.at(-1)?.id ?? null;
|
|
16
|
+
const [dismissedOf, setDismissedOf] = useState<string | null>(null);
|
|
17
|
+
const dismiss = useCallback(() => setDismissedOf(lastMessageId), [lastMessageId]);
|
|
18
|
+
return { questions: dismissedOf === lastMessageId ? [] : asked, dismiss };
|
|
19
|
+
}
|
|
@@ -18,6 +18,10 @@
|
|
|
18
18
|
* the person says a few words that are not the answer echoing back, the turn
|
|
19
19
|
* is aborted — playback stops, the chat request is cancelled — and what they
|
|
20
20
|
* are saying becomes the next utterance.
|
|
21
|
+
* - **A voice that cannot speak ends the call.** When a clip cannot be made
|
|
22
|
+
* (`synthesisError`, typed from the route's code) or played, the turn stops
|
|
23
|
+
* speaking, its answer finishes streaming into the transcript, and the call
|
|
24
|
+
* ends with the reason as `errorCode` — never a call that listens on, mute.
|
|
21
25
|
*
|
|
22
26
|
* It replaced a LiveKit room (`POST /v1/voice/token`) whose realtime model and
|
|
23
27
|
* transcription called providers directly; Alia's only inference path is
|
|
@@ -43,11 +47,17 @@ import type {
|
|
|
43
47
|
} from '../lib/speech-recognition-types';
|
|
44
48
|
import {
|
|
45
49
|
speechFailureCode,
|
|
50
|
+
synthesisFailureCode,
|
|
46
51
|
VOICE_ERROR_MESSAGES,
|
|
47
52
|
type VoiceErrorCode,
|
|
48
53
|
} from '../lib/speech-messages';
|
|
49
54
|
import { defaultSpeechLanguage } from '../lib/speech-language';
|
|
50
|
-
import {
|
|
55
|
+
import {
|
|
56
|
+
playSpeechClip,
|
|
57
|
+
requestSpeechClip,
|
|
58
|
+
SpeechClipError,
|
|
59
|
+
type ProductVoice,
|
|
60
|
+
} from '../lib/speech-synthesis';
|
|
51
61
|
import { VoiceLevelChannel, type VoiceLevelSource } from '../lib/voice-levels';
|
|
52
62
|
import {
|
|
53
63
|
createAliaVoiceTurnSender,
|
|
@@ -133,7 +143,15 @@ interface ActiveTurn {
|
|
|
133
143
|
closed: boolean;
|
|
134
144
|
/** Wakes the player when a clip is queued or the answer ends. */
|
|
135
145
|
wake: (() => void) | null;
|
|
136
|
-
|
|
146
|
+
/**
|
|
147
|
+
* Why this turn stopped speaking: a clip that could not be made (`synthesis`
|
|
148
|
+
* holds what the route said) or one the device could not play. The call ends
|
|
149
|
+
* with `code` once the answer has finished streaming.
|
|
150
|
+
*/
|
|
151
|
+
speechFailure: {
|
|
152
|
+
readonly code: VoiceErrorCode;
|
|
153
|
+
readonly synthesis: SpeechClipError | null;
|
|
154
|
+
} | null;
|
|
137
155
|
}
|
|
138
156
|
|
|
139
157
|
// ============== HOOK ==============
|
|
@@ -156,8 +174,10 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
156
174
|
const [callFailure, setCallFailure] = useState<{ code: VoiceErrorCode; message: string } | null>(
|
|
157
175
|
null,
|
|
158
176
|
);
|
|
159
|
-
/** A turn that failed without ending the call: no answer
|
|
177
|
+
/** A turn that failed without ending the call: no answer came back. */
|
|
160
178
|
const [turnErrorCode, setTurnErrorCode] = useState<VoiceErrorCode | null>(null);
|
|
179
|
+
/** The clip that could not be made, when that is what ended the call. */
|
|
180
|
+
const [synthesisError, setSynthesisError] = useState<SpeechClipError | null>(null);
|
|
161
181
|
const [messages, setMessagesState] = useState<VoiceMessage[]>([]);
|
|
162
182
|
const [currentSpeaker, setCurrentSpeaker] = useState<'primary' | 'user' | null>(null);
|
|
163
183
|
|
|
@@ -307,11 +327,23 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
307
327
|
|
|
308
328
|
// ============== SPEAKING ==============
|
|
309
329
|
|
|
330
|
+
/** The first reason a turn could not speak is the one the call ends with. */
|
|
331
|
+
const stopSpeaking = useCallback(
|
|
332
|
+
(turn: ActiveTurn, code: VoiceErrorCode, synthesis: SpeechClipError | null) => {
|
|
333
|
+
turn.speechFailure ??= { code, synthesis };
|
|
334
|
+
},
|
|
335
|
+
[],
|
|
336
|
+
);
|
|
337
|
+
|
|
310
338
|
const synthesize = useCallback(
|
|
311
339
|
(turn: ActiveTurn, text: string): Promise<string | null> => {
|
|
312
340
|
const token = getToken();
|
|
313
341
|
if (token === null) {
|
|
314
|
-
|
|
342
|
+
const failure = new SpeechClipError(VOICE_ERROR_MESSAGES['not-authenticated'], {
|
|
343
|
+
status: null,
|
|
344
|
+
code: 'NOT_AUTHENTICATED',
|
|
345
|
+
});
|
|
346
|
+
stopSpeaking(turn, synthesisFailureCode(failure), failure);
|
|
315
347
|
return Promise.resolve(null);
|
|
316
348
|
}
|
|
317
349
|
const { apiUrl: url, speechModel: model, voicePref: voice } = configRef.current;
|
|
@@ -323,17 +355,28 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
323
355
|
voice,
|
|
324
356
|
signal: turn.controller.signal,
|
|
325
357
|
}).catch((caught: unknown) => {
|
|
326
|
-
if (
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
358
|
+
if (turn.controller.signal.aborted) return null;
|
|
359
|
+
const failure =
|
|
360
|
+
caught instanceof SpeechClipError
|
|
361
|
+
? caught
|
|
362
|
+
: new SpeechClipError(errorMessage(caught, 'TTS failed'), { status: null, code: null });
|
|
363
|
+
console.error('[useVoiceRoom] Speech synthesis failed:', {
|
|
364
|
+
status: failure.status,
|
|
365
|
+
code: failure.code,
|
|
366
|
+
requestId: failure.requestId,
|
|
367
|
+
});
|
|
368
|
+
stopSpeaking(turn, synthesisFailureCode(failure), failure);
|
|
330
369
|
return null;
|
|
331
370
|
});
|
|
332
371
|
},
|
|
333
|
-
[getToken],
|
|
372
|
+
[getToken, stopSpeaking],
|
|
334
373
|
);
|
|
335
374
|
|
|
336
|
-
/**
|
|
375
|
+
/**
|
|
376
|
+
* Play the turn's clips in order as they arrive, fetching one ahead. Returns
|
|
377
|
+
* early, without skipping to a later sentence, once a clip cannot be made or
|
|
378
|
+
* played.
|
|
379
|
+
*/
|
|
337
380
|
const playTurn = useCallback(
|
|
338
381
|
async (turn: ActiveTurn): Promise<void> => {
|
|
339
382
|
const signal = turn.controller.signal;
|
|
@@ -352,18 +395,25 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
352
395
|
|
|
353
396
|
const uri = await item.clip;
|
|
354
397
|
if (signal.aborted) return;
|
|
355
|
-
if (uri === null)
|
|
398
|
+
if (uri === null) {
|
|
399
|
+
setAgent('thinking');
|
|
400
|
+
levels.setPlayback(0);
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
356
403
|
turn.speaking = `${turn.speaking} ${item.text}`;
|
|
357
404
|
setAgent('speaking');
|
|
358
405
|
try {
|
|
359
406
|
await playSpeechClip(uri, { signal, onLevel: (level) => levels.setPlayback(level) });
|
|
360
407
|
} catch (caught: unknown) {
|
|
408
|
+
if (signal.aborted) return;
|
|
361
409
|
console.error('[useVoiceRoom] Playback failed:', caught);
|
|
362
|
-
turn
|
|
410
|
+
stopSpeaking(turn, 'not-played', null);
|
|
411
|
+
setAgent('thinking');
|
|
412
|
+
return;
|
|
363
413
|
}
|
|
364
414
|
}
|
|
365
415
|
},
|
|
366
|
-
[levels, setAgent, synthesize],
|
|
416
|
+
[levels, setAgent, stopSpeaking, synthesize],
|
|
367
417
|
);
|
|
368
418
|
|
|
369
419
|
const queueChunks = useCallback((turn: ActiveTurn, final: boolean) => {
|
|
@@ -545,7 +595,7 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
545
595
|
clips: [],
|
|
546
596
|
closed: false,
|
|
547
597
|
wake: null,
|
|
548
|
-
|
|
598
|
+
speechFailure: null,
|
|
549
599
|
};
|
|
550
600
|
turnRef.current = turn;
|
|
551
601
|
setAgent('thinking');
|
|
@@ -588,6 +638,15 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
588
638
|
await playing;
|
|
589
639
|
if (signal.aborted || turnRef.current !== turn) return;
|
|
590
640
|
|
|
641
|
+
// The answer is in the transcript; a call that cannot say it ends here,
|
|
642
|
+
// rather than listening on in silence.
|
|
643
|
+
const speechFailure = turn.speechFailure;
|
|
644
|
+
if (speechFailure !== null) {
|
|
645
|
+
if (mountedRef.current) setSynthesisError(speechFailure.synthesis);
|
|
646
|
+
fail(speechFailure.code);
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
649
|
+
|
|
591
650
|
turnRef.current = null;
|
|
592
651
|
levels.setPlayback(0);
|
|
593
652
|
// The session that listened through the answer has heard the answer; a
|
|
@@ -596,8 +655,6 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
596
655
|
if (turn.text.trim() === '') {
|
|
597
656
|
updateMessages((previous) => previous.filter((message) => message.id !== assistantId));
|
|
598
657
|
if (mountedRef.current) setTurnErrorCode('no-answer');
|
|
599
|
-
} else if (turn.synthesisFailed) {
|
|
600
|
-
if (mountedRef.current) setTurnErrorCode('not-played');
|
|
601
658
|
}
|
|
602
659
|
setAgent('listening');
|
|
603
660
|
startListeningRef.current();
|
|
@@ -613,6 +670,7 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
613
670
|
if (phaseRef.current !== 'off') return;
|
|
614
671
|
setCallFailure(null);
|
|
615
672
|
setTurnErrorCode(null);
|
|
673
|
+
setSynthesisError(null);
|
|
616
674
|
setRoomState('connecting');
|
|
617
675
|
|
|
618
676
|
if (!isSpeechRecognitionAvailable()) {
|
|
@@ -669,6 +727,7 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
669
727
|
setAgentStateValue('idle');
|
|
670
728
|
setCallFailure(null);
|
|
671
729
|
setTurnErrorCode(null);
|
|
730
|
+
setSynthesisError(null);
|
|
672
731
|
setIsMuted(false);
|
|
673
732
|
setMessagesState([]);
|
|
674
733
|
setCurrentSpeaker(null);
|
|
@@ -718,6 +777,12 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
718
777
|
errorCode: callFailure?.code ?? null,
|
|
719
778
|
turnError: turnErrorCode === null ? null : VOICE_ERROR_MESSAGES[turnErrorCode],
|
|
720
779
|
turnErrorCode,
|
|
780
|
+
/**
|
|
781
|
+
* When the call ended because a clip could not be made: the route's HTTP
|
|
782
|
+
* status and typed code (`SpeechClipError`). `errorCode` says the same as a
|
|
783
|
+
* `VoiceErrorCode` an app can translate.
|
|
784
|
+
*/
|
|
785
|
+
synthesisError,
|
|
721
786
|
messages,
|
|
722
787
|
currentSpeaker,
|
|
723
788
|
connect,
|