@alia.onl/sdk 13.1.0 → 15.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +190 -0
  2. package/README.md +83 -20
  3. package/package.json +8 -4
  4. package/src/components/AliaAvatarIcon.tsx +30 -0
  5. package/src/components/AliaChatContent.tsx +57 -107
  6. package/src/components/AliaChatMessageList.tsx +2 -10
  7. package/src/components/AliaChatScreen.tsx +28 -25
  8. package/src/components/AliaChatSheet.tsx +30 -8
  9. package/src/components/AliaComposer.tsx +382 -0
  10. package/src/components/ComposerDropZone.tsx +235 -0
  11. package/src/components/voice/VoiceControls.tsx +28 -4
  12. package/src/components/voice/VoiceSession.tsx +23 -7
  13. package/src/hooks/useAliaActivity.ts +31 -0
  14. package/src/hooks/useAliaChat.ts +14 -3
  15. package/src/hooks/useAmbientWave.ts +3 -3
  16. package/src/hooks/useAttachmentIntake.ts +401 -0
  17. package/src/hooks/useAudioLevelMonitor.ts +2 -2
  18. package/src/hooks/useDictation.ts +295 -0
  19. package/src/hooks/useFocusWhenPresented.ts +41 -0
  20. package/src/hooks/usePendingQuestions.ts +19 -0
  21. package/src/hooks/useTTS.ts +24 -63
  22. package/src/hooks/useVoiceRoom.ts +148 -671
  23. package/src/index.ts +19 -3
  24. package/src/lib/activity-socket.ts +102 -0
  25. package/src/lib/activity-store.ts +53 -0
  26. package/src/lib/api-url.ts +2 -0
  27. package/src/lib/attachment-intake.ts +154 -0
  28. package/src/lib/attachment-tiles.ts +74 -0
  29. package/src/lib/audio-level.ts +2 -2
  30. package/src/lib/dictation-recorder-types.ts +37 -0
  31. package/src/lib/dictation-recorder.native.ts +90 -0
  32. package/src/lib/dictation-recorder.ts +131 -0
  33. package/src/lib/dictation.ts +208 -0
  34. package/src/lib/labels.tsx +50 -1
  35. package/src/lib/pcm.ts +57 -0
  36. package/src/lib/speech-language.ts +1 -1
  37. package/src/lib/speech-messages.ts +43 -65
  38. package/src/lib/speech-synthesis.ts +104 -9
  39. package/src/lib/turn-state.ts +15 -0
  40. package/src/lib/voice-audio-engine-types.ts +59 -0
  41. package/src/lib/voice-audio-engine.native.ts +232 -0
  42. package/src/lib/voice-audio-engine.ts +350 -0
  43. package/src/lib/voice-call.ts +539 -0
  44. package/src/lib/voice-levels.ts +7 -3
  45. package/src/lib/voice-protocol.ts +309 -0
  46. package/src/tools.ts +1 -0
  47. package/src/types.ts +63 -4
  48. package/src/voice.ts +31 -10
  49. package/tsconfig.json +2 -0
  50. package/src/hooks/useSpeechToText.ts +0 -268
  51. package/src/lib/speech-recognition-types.ts +0 -108
  52. package/src/lib/speech-recognition.native.ts +0 -237
  53. package/src/lib/speech-recognition.ts +0 -308
  54. package/src/lib/voice-text.ts +0 -198
  55. package/src/lib/voice-turn.ts +0 -89
@@ -1,725 +1,202 @@
1
1
  /**
2
- * A voice conversation, run on the device as a turn loop.
2
+ * A voice call: speech to speech through Alia's `/v1/voice/session`
3
+ * (ADR 0017).
3
4
  *
4
- * listening ──(the person stops talking)──▶ thinking ──(first sentence)──▶ speaking
5
- * ▲ │ │
6
- * └──────────(answer spoken, or the person talks over it)──────────────────┘
5
+ * Alia holds the realtime session with Kaana — instructions, memory, tools
6
+ * (run in Alia, never here), the transcript stored in the conversation, credits
7
+ * and typed errors. Turn detection and barge-in are the model's, on the server.
8
+ * This hook captures and plays PCM through the platform's audio engine
9
+ * (`lib/voice-audio-engine*`) and exposes a state it can prove (`callState`,
10
+ * see `VoiceCallState`), with `roomState` / `agentState` as the coarse
11
+ * projections the call bar, the ambient field and the sound effects read.
7
12
  *
8
- * - **Listening** is on-device speech recognition (`lib/speech-recognition*`).
9
- * An utterance ends after `endOfUtteranceMs` without a new word.
10
- * - **Thinking** sends the utterance through the ordinary chat path — a
11
- * `VoiceTurnSender`, by default `POST /v1/chat/completions` marked
12
- * `responseMode: 'voice'` — and streams the answer into the transcript.
13
- * - **Speaking** synthesizes the answer through `POST /v1/audio/speech`
14
- * sentence by sentence as it streams, so the first sentence plays while the
15
- * rest is still being written, and the next clip is fetched while the
16
- * current one plays.
17
- * - **Barge-in:** the microphone stays open while Alia thinks and speaks. When
18
- * the person says a few words that are not the answer echoing back, the turn
19
- * is aborted — playback stops, the chat request is cancelled — and what they
20
- * are saying becomes the next utterance.
21
- *
22
- * It replaced a LiveKit room (`POST /v1/voice/token`) whose realtime model and
23
- * transcription called providers directly; Alia's only inference path is
24
- * Alia → Oxy → Kaana, which serves chat and speech but no realtime session or
25
- * transcription. The public shape — `connect` / `disconnect`, `roomState`,
26
- * `agentState`, `messages`, `toggleMute` — is unchanged, so its consumers did
27
- * not change with it.
13
+ * Nothing is recognized on the device: there is no browser or OS speech
14
+ * recognizer anywhere in a call.
28
15
  */
29
16
 
30
- import { useState, useCallback, useRef, useEffect, useMemo } from 'react';
17
+ import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
31
18
  import { useOxy } from '@oxy.so/services/ui/client';
32
- import { errorMessage } from '../lib/utils';
33
- import type { RoomState, AgentState, VoiceMessage } from '../types';
34
- import {
35
- isSpeechRecognitionAvailable,
36
- chooseSpeechRecognizer,
37
- requestSpeechRecognitionPermission,
38
- startSpeechRecognition,
39
- } from '../lib/speech-recognition';
40
- import type {
41
- SpeechRecognitionFailure,
42
- SpeechRecognitionSession,
43
- } from '../lib/speech-recognition-types';
44
- import {
45
- speechFailureCode,
46
- VOICE_ERROR_MESSAGES,
47
- type VoiceErrorCode,
48
- } from '../lib/speech-messages';
19
+ import type { AgentState, RoomState, VoiceCallState, VoiceMessage } from '../types';
20
+ import { VOICE_ERROR_MESSAGES, type VoiceErrorCode } from '../lib/speech-messages';
49
21
  import { defaultSpeechLanguage } from '../lib/speech-language';
50
- import { playSpeechClip, requestSpeechClip, type ProductVoice } from '../lib/speech-synthesis';
22
+ import { createVoiceAudioEngine } from '../lib/voice-audio-engine';
23
+ import type { VoiceAudioEngine } from '../lib/voice-audio-engine-types';
24
+ import { VoiceCall, type VoiceCallSnapshot, type VoiceSocketFactory } from '../lib/voice-call';
51
25
  import { VoiceLevelChannel, type VoiceLevelSource } from '../lib/voice-levels';
52
- import {
53
- createAliaVoiceTurnSender,
54
- type VoiceTurnMessage,
55
- type VoiceTurnSender,
56
- } from '../lib/voice-turn';
57
- import {
58
- dropEchoPrefix,
59
- isInterruption,
60
- stripTitleTags,
61
- stripTitleTagsPartial,
62
- takeSpeechChunks,
63
- } from '../lib/voice-text';
26
+ import type { ProductVoice, VoiceCloseReason } from '../lib/voice-protocol';
64
27
 
65
28
  const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
66
29
 
67
- // ============== OPTIONS ==============
68
-
69
30
  export interface UseVoiceRoomOptions {
70
31
  apiUrl?: string;
71
- /** Which of the product's two voices answers. Sent to `/v1/audio/speech` as is. */
72
- voicePreference?: ProductVoice;
73
- /** Bearer for speech synthesis; defaults to the surrounding Oxy session's. */
74
- accessToken?: string;
75
- /**
76
- * Speech model for the answers. Omitted, the request carries no `model` and
77
- * the server's own speech model answers.
78
- */
79
- model?: string;
80
32
  /**
81
- * Chat model (`publisher/model`) the default sender asks for; omitted, the
82
- * server's default. Ignored when `sendTurn`
83
- * is given — that sender carries its own conversation's choice.
84
- */
85
- chatModel?: string;
86
- /**
87
- * The agent this call belongs to, when it is a thread with one.
88
- *
89
- * Without it the call is ordinary Alia. Only the default sender reads it; a
90
- * custom `sendTurn` is already bound to its conversation's agent.
33
+ * The conversation the call is stored in. Omitted, every call starts a
34
+ * conversation of its own (a fresh id per `connect`).
91
35
  */
36
+ conversationId?: string;
37
+ /** The agent this call talks to; omitted, Alia. */
92
38
  agentId?: string;
93
- /** Language to recognize (BCP-47). Defaults to the platform's. */
39
+ /** Which of the product's two voices answers. */
40
+ voicePreference?: ProductVoice;
41
+ /** Bearer for the call; defaults to the surrounding Oxy session's. */
42
+ accessToken?: string;
43
+ /** Language spoken (BCP-47). Defaults to the platform's. */
94
44
  lang?: string;
95
- /**
96
- * How each turn reaches Alia. Defaults to `createAliaVoiceTurnSender`. The
97
- * Alia app passes its conversation's own send, so a call's turns are that
98
- * conversation's turns — persisted and shown like typed ones.
99
- */
100
- sendTurn?: VoiceTurnSender;
101
- /** Silence, in ms, that ends an utterance. */
102
- endOfUtteranceMs?: number;
103
- /** Let the person interrupt by talking. On by default. */
104
- bargeIn?: boolean;
45
+ /** Alia stored messages of the call: refresh the thread. */
46
+ onMessagesSaved?: (conversationId: string, messageIds: readonly string[]) => void;
47
+ /** A tool filed an action for the person's OK; its card is in the conversation. */
48
+ onApprovalRequested?: (toolName: string) => void;
49
+ /** Test seams: the device's audio and the socket. */
50
+ createAudioEngine?: () => VoiceAudioEngine;
51
+ openSocket?: VoiceSocketFactory;
105
52
  }
106
53
 
107
- // ============== TUNING ==============
108
-
109
- const DEFAULT_END_OF_UTTERANCE_MS = 1_100;
110
- /** How long a requested stop may take to deliver the final words. */
111
- const COMMIT_TIMEOUT_MS = 1_500;
112
- /** Words needed before talking over the answer counts as an interruption. */
113
- const BARGE_IN_MIN_WORDS = 2;
114
- /** A recognizer that keeps ending within this of starting is failing, not idling. */
115
- const RAPID_END_MS = 400;
116
- const MAX_RAPID_ENDS = 5;
54
+ const IDLE: VoiceCallSnapshot = {
55
+ state: { kind: 'idle' },
56
+ messages: [],
57
+ isMuted: false,
58
+ endingSoonMs: null,
59
+ model: null,
60
+ };
117
61
 
118
- // ============== INTERNAL STATE ==============
62
+ const openWebSocket: VoiceSocketFactory = (url) =>
63
+ new WebSocket(url) as unknown as ReturnType<VoiceSocketFactory>;
119
64
 
120
- type Phase = 'off' | 'listening' | 'thinking' | 'speaking';
65
+ function newConversationId(): string {
66
+ const random = globalThis.crypto?.randomUUID?.();
67
+ return random ?? `voice-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
68
+ }
121
69
 
122
- interface ActiveTurn {
123
- readonly controller: AbortController;
124
- readonly assistantId: string;
125
- /** Characters of the answer already handed to the speech queue. */
126
- consumed: number;
127
- chunkCount: number;
128
- /** The answer so far, title tags stripped and whitespace kept (see `stripTitleTags`). */
129
- text: string;
130
- /** Everything this turn has said aloud so far, for telling echo from the person. */
131
- speaking: string;
132
- readonly clips: Array<{ readonly text: string; clip: Promise<string | null> | null }>;
133
- closed: boolean;
134
- /** Wakes the player when a clip is queued or the answer ends. */
135
- wake: (() => void) | null;
136
- synthesisFailed: boolean;
70
+ /** The coarse connection state the bar and the wave read. */
71
+ export function roomStateOf(state: VoiceCallState): RoomState {
72
+ switch (state.kind) {
73
+ case 'idle':
74
+ case 'ended':
75
+ return 'disconnected';
76
+ case 'connecting':
77
+ return 'connecting';
78
+ case 'reconnecting':
79
+ return 'reconnecting';
80
+ case 'error':
81
+ return 'error';
82
+ default:
83
+ return 'connected';
84
+ }
137
85
  }
138
86
 
139
- // ============== HOOK ==============
87
+ /** Who has the turn, coarsely. */
88
+ export function agentStateOf(state: VoiceCallState): AgentState {
89
+ switch (state.kind) {
90
+ case 'listening':
91
+ case 'user-speaking':
92
+ return 'listening';
93
+ case 'thinking':
94
+ case 'using-tool':
95
+ return 'thinking';
96
+ case 'speaking':
97
+ return 'speaking';
98
+ default:
99
+ return 'idle';
100
+ }
101
+ }
140
102
 
141
103
  export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
142
- const apiUrl = options.apiUrl || API_URL;
143
- const voicePref: ProductVoice = options.voicePreference ?? 'female';
144
- const speechModel = options.model;
145
- const endOfUtteranceMs = options.endOfUtteranceMs ?? DEFAULT_END_OF_UTTERANCE_MS;
146
- const bargeIn = options.bargeIn ?? true;
147
-
148
- const [roomState, setRoomState] = useState<RoomState>('disconnected');
149
- const [agentState, setAgentStateValue] = useState<AgentState>('idle');
150
- const [isMuted, setIsMuted] = useState(false);
151
- /**
152
- * Why the call ended, as a code and as the sentence to show. The sentence is
153
- * the code's English copy, except for `turn-failed`, which carries what the
154
- * turn sender threw.
155
- */
156
- const [callFailure, setCallFailure] = useState<{ code: VoiceErrorCode; message: string } | null>(
157
- null,
158
- );
159
- /** A turn that failed without ending the call: no answer, or no audio for it. */
160
- const [turnErrorCode, setTurnErrorCode] = useState<VoiceErrorCode | null>(null);
161
- const [messages, setMessagesState] = useState<VoiceMessage[]>([]);
162
- const [currentSpeaker, setCurrentSpeaker] = useState<'primary' | 'user' | null>(null);
163
-
164
104
  const { oxyServices } = useOxy();
165
-
166
105
  const levels = useMemo(() => new VoiceLevelChannel(), []);
167
-
168
- // Everything the loop reads from callbacks lives in refs: recognizer events
169
- // arrive outside React, and a value closed over at `connect` would be stale.
170
- const phaseRef = useRef<Phase>('off');
171
- const mutedRef = useRef(false);
106
+ const [snapshot, setSnapshot] = useState<VoiceCallSnapshot>(IDLE);
107
+ const callRef = useRef<VoiceCall | null>(null);
172
108
  const mountedRef = useRef(true);
173
- const messagesRef = useRef<VoiceMessage[]>([]);
174
- const sessionRef = useRef<SpeechRecognitionSession | null>(null);
175
- /** The recognizer and tag the call listens with, chosen when it connects. */
176
- const recognizerRef = useRef<{ lang: string; service?: string }>({ lang: '' });
177
- const heardRef = useRef('');
178
- const committingRef = useRef(false);
179
- const silenceTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
180
- const commitTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
181
- const turnRef = useRef<ActiveTurn | null>(null);
182
- const draftIdRef = useRef<string | null>(null);
183
- /** What the call had said when the person cut in; this session's echo of it is not theirs. */
184
- const echoContextRef = useRef('');
185
- const sessionStartedAtRef = useRef(0);
186
- const rapidEndsRef = useRef(0);
187
- const sessionFailedRef = useRef<SpeechRecognitionFailure | null>(null);
188
- const msgIdRef = useRef(0);
189
- const sessionPrefixRef = useRef(`vm-${Date.now().toString(36)}`);
190
-
191
- const config = {
192
- apiUrl,
193
- voicePref,
194
- speechModel,
195
- endOfUtteranceMs,
196
- bargeIn,
197
- lang: options.lang ?? defaultSpeechLanguage(),
198
- accessToken: options.accessToken,
199
- sendTurn:
200
- options.sendTurn ??
201
- createAliaVoiceTurnSender({
202
- oxyServices,
203
- apiUrl,
204
- ...(options.chatModel === undefined ? {} : { model: options.chatModel }),
205
- ...(options.agentId === undefined ? {} : { agentId: options.agentId }),
206
- }),
207
- };
208
- const configRef = useRef(config);
209
- configRef.current = config;
210
-
211
- // ============== STATE HELPERS ==============
212
-
213
- const setAgent = useCallback((phase: Phase) => {
214
- phaseRef.current = phase;
215
- if (!mountedRef.current) return;
216
- setAgentStateValue(phase === 'off' ? 'idle' : phase);
217
- setCurrentSpeaker(phase === 'thinking' || phase === 'speaking' ? 'primary' : null);
218
- }, []);
219
-
220
- const updateMessages = useCallback((update: (previous: VoiceMessage[]) => VoiceMessage[]) => {
221
- messagesRef.current = update(messagesRef.current);
222
- if (mountedRef.current) setMessagesState(messagesRef.current);
223
- }, []);
224
-
225
- const nextId = (): string => {
226
- msgIdRef.current += 1;
227
- return `${sessionPrefixRef.current}-${msgIdRef.current}`;
228
- };
229
-
230
- const patchMessage = useCallback(
231
- (id: string, patch: Partial<VoiceMessage>) => {
232
- updateMessages((previous) =>
233
- previous.map((message) => (message.id === id ? { ...message, ...patch } : message)),
234
- );
235
- },
236
- [updateMessages],
237
- );
109
+ const optionsRef = useRef(options);
110
+ optionsRef.current = options;
238
111
 
239
112
  const getToken = useCallback((): string | null => {
240
- const { accessToken } = configRef.current;
113
+ const { accessToken } = optionsRef.current;
241
114
  if (accessToken) return accessToken;
242
- return oxyServices.session.accessToken;
115
+ return oxyServices.session.accessToken ?? null;
243
116
  }, [oxyServices]);
244
117
 
245
- const clearTimers = (): void => {
246
- if (silenceTimerRef.current !== null) clearTimeout(silenceTimerRef.current);
247
- if (commitTimerRef.current !== null) clearTimeout(commitTimerRef.current);
248
- silenceTimerRef.current = null;
249
- commitTimerRef.current = null;
250
- };
251
-
252
- // ============== TEARDOWN ==============
253
-
254
- const endTurn = useCallback(
255
- (turn: ActiveTurn) => {
256
- turn.controller.abort();
257
- turn.closed = true;
258
- turn.wake?.();
259
- if (turnRef.current === turn) turnRef.current = null;
260
- const message = messagesRef.current.find((candidate) => candidate.id === turn.assistantId);
261
- if (message?.isStreaming) {
262
- if (message.content.trim() === '') {
263
- updateMessages((previous) =>
264
- previous.filter((candidate) => candidate.id !== turn.assistantId),
265
- );
266
- } else {
267
- patchMessage(turn.assistantId, { isStreaming: false });
268
- }
269
- }
270
- levels.setPlayback(0);
271
- },
272
- [levels, patchMessage, updateMessages],
273
- );
274
-
275
- const discardDraft = useCallback(() => {
276
- const draftId = draftIdRef.current;
277
- draftIdRef.current = null;
278
- if (draftId !== null)
279
- updateMessages((previous) => previous.filter((message) => message.id !== draftId));
280
- }, [updateMessages]);
281
-
282
- /** Stop everything; the caller decides what state the room is left in. */
283
- const teardown = useCallback(() => {
284
- phaseRef.current = 'off';
285
- clearTimers();
286
- committingRef.current = false;
287
- const turn = turnRef.current;
288
- if (turn !== null) endTurn(turn);
289
- sessionRef.current?.abort();
290
- sessionRef.current = null;
291
- heardRef.current = '';
292
- levels.reset();
293
- }, [endTurn, levels]);
294
-
295
- const fail = useCallback(
296
- (code: VoiceErrorCode, message: string = VOICE_ERROR_MESSAGES[code]) => {
297
- teardown();
298
- discardDraft();
299
- if (!mountedRef.current) return;
300
- setCallFailure({ code, message });
301
- setRoomState('error');
302
- setAgentStateValue('idle');
303
- setCurrentSpeaker(null);
304
- },
305
- [discardDraft, teardown],
306
- );
307
-
308
- // ============== SPEAKING ==============
309
-
310
- const synthesize = useCallback(
311
- (turn: ActiveTurn, text: string): Promise<string | null> => {
312
- const token = getToken();
313
- if (token === null) {
314
- turn.synthesisFailed = true;
315
- return Promise.resolve(null);
316
- }
317
- const { apiUrl: url, speechModel: model, voicePref: voice } = configRef.current;
318
- return requestSpeechClip({
319
- apiUrl: url,
320
- token,
321
- model,
322
- input: text,
323
- voice,
324
- signal: turn.controller.signal,
325
- }).catch((caught: unknown) => {
326
- if (!turn.controller.signal.aborted) {
327
- console.error('[useVoiceRoom] Speech synthesis failed:', caught);
328
- turn.synthesisFailed = true;
329
- }
330
- return null;
331
- });
332
- },
333
- [getToken],
334
- );
335
-
336
- /** Play the turn's clips in order as they arrive, fetching one ahead. */
337
- const playTurn = useCallback(
338
- async (turn: ActiveTurn): Promise<void> => {
339
- const signal = turn.controller.signal;
340
- for (let index = 0; ; index += 1) {
341
- while (index >= turn.clips.length && !turn.closed) {
342
- await new Promise<void>((resolve) => {
343
- turn.wake = resolve;
344
- });
345
- turn.wake = null;
346
- }
347
- if (signal.aborted || index >= turn.clips.length) return;
348
- const item = turn.clips[index];
349
- item.clip ??= synthesize(turn, item.text);
350
- const next = turn.clips[index + 1];
351
- if (next !== undefined) next.clip ??= synthesize(turn, next.text);
352
-
353
- const uri = await item.clip;
354
- if (signal.aborted) return;
355
- if (uri === null) continue;
356
- turn.speaking = `${turn.speaking} ${item.text}`;
357
- setAgent('speaking');
358
- try {
359
- await playSpeechClip(uri, { signal, onLevel: (level) => levels.setPlayback(level) });
360
- } catch (caught: unknown) {
361
- console.error('[useVoiceRoom] Playback failed:', caught);
362
- turn.synthesisFailed = true;
363
- }
364
- }
365
- },
366
- [levels, setAgent, synthesize],
367
- );
368
-
369
- const queueChunks = useCallback((turn: ActiveTurn, final: boolean) => {
370
- const pending = turn.text.slice(turn.consumed);
371
- const { chunks, rest } = takeSpeechChunks(pending, { final, isFirst: turn.chunkCount === 0 });
372
- turn.consumed = turn.text.length - rest.length;
373
- for (const text of chunks) {
374
- turn.chunkCount += 1;
375
- // The next clip is fetched now if the player is waiting on it; the
376
- // player fetches the rest one ahead of what it is playing.
377
- turn.clips.push({ text, clip: null });
378
- }
379
- if (chunks.length > 0) turn.wake?.();
380
- }, []);
381
-
382
- // ============== LISTENING ==============
383
-
384
- // Declared through refs so the recognizer callbacks and the turn runner can
385
- // call each other without a cycle in their dependency lists.
386
- const startListeningRef = useRef<() => void>(() => undefined);
387
- const runTurnRef = useRef<(text: string) => void>(() => undefined);
388
-
389
- const commitUtterance = useCallback(() => {
390
- const session = sessionRef.current;
391
- if (session === null || committingRef.current) return;
392
- committingRef.current = true;
393
- session.stop();
394
- commitTimerRef.current = setTimeout(() => {
395
- commitTimerRef.current = null;
396
- sessionRef.current?.abort();
397
- }, COMMIT_TIMEOUT_MS);
398
- }, []);
399
-
400
- const showDraft = useCallback(
401
- (transcript: string) => {
402
- const draftId = draftIdRef.current;
403
- if (draftId !== null) {
404
- patchMessage(draftId, { content: transcript });
405
- return;
406
- }
407
- const id = nextId();
408
- draftIdRef.current = id;
409
- updateMessages((previous) => [
410
- ...previous,
411
- { id, role: 'user', content: transcript, timestamp: Date.now(), isStreaming: true },
412
- ]);
413
- if (mountedRef.current) setCurrentSpeaker('user');
414
- },
415
- [patchMessage, updateMessages],
416
- );
417
-
418
- const startListening = useCallback(() => {
419
- if (phaseRef.current === 'off' || mutedRef.current || sessionRef.current !== null) return;
420
- heardRef.current = '';
421
- committingRef.current = false;
422
- sessionFailedRef.current = null;
423
- echoContextRef.current = '';
424
- sessionStartedAtRef.current = Date.now();
425
-
426
- sessionRef.current = startSpeechRecognition(
427
- { ...recognizerRef.current, echoCancellation: true },
428
- {
429
- onResult: ({ transcript }) => {
430
- if (phaseRef.current === 'off' || committingRef.current) return;
431
-
432
- const turn = turnRef.current;
433
- if (turn !== null) {
434
- // Alia is thinking or speaking: this is the person cutting in, or
435
- // the answer coming back through the microphone.
436
- heardRef.current = '';
437
- if (!configRef.current.bargeIn) return;
438
- if (!isInterruption(transcript, turn.speaking, BARGE_IN_MIN_WORDS)) return;
439
- echoContextRef.current = turn.speaking;
440
- endTurn(turn);
441
- setAgent('listening');
442
- }
443
-
444
- const utterance = dropEchoPrefix(transcript, echoContextRef.current);
445
- heardRef.current = utterance;
446
- if (utterance === '') return;
447
- showDraft(utterance);
448
- if (silenceTimerRef.current !== null) clearTimeout(silenceTimerRef.current);
449
- silenceTimerRef.current = setTimeout(() => {
450
- silenceTimerRef.current = null;
451
- commitUtterance();
452
- }, configRef.current.endOfUtteranceMs);
453
- },
454
- onLevel: (level) => {
455
- levels.setCapture(mutedRef.current ? 0 : level);
456
- },
457
- onError: (failure) => {
458
- if (failure.code === 'no-speech' || failure.code === 'aborted') return;
459
- sessionFailedRef.current = failure;
460
- },
461
- onEnd: () => {
462
- sessionRef.current = null;
463
- clearTimers();
464
- const committing = committingRef.current;
465
- committingRef.current = false;
466
- const heard = heardRef.current.trim();
467
- heardRef.current = '';
468
- if (phaseRef.current === 'off') return;
469
-
470
- const failure = sessionFailedRef.current;
471
- if (failure !== null && failure.code !== 'other') {
472
- fail(speechFailureCode(failure) ?? 'speech-failed');
473
- return;
474
- }
475
-
476
- // An utterance in progress when the engine ended is still an utterance.
477
- if (
478
- (committing || draftIdRef.current !== null) &&
479
- heard !== '' &&
480
- turnRef.current === null
481
- ) {
482
- rapidEndsRef.current = 0;
483
- runTurnRef.current(heard);
484
- return;
485
- }
486
-
487
- if (Date.now() - sessionStartedAtRef.current < RAPID_END_MS) {
488
- rapidEndsRef.current += 1;
489
- if (rapidEndsRef.current >= MAX_RAPID_ENDS) {
490
- fail(speechFailureCode(failure ?? { code: 'other' }) ?? 'speech-failed');
491
- return;
492
- }
493
- } else {
494
- rapidEndsRef.current = 0;
495
- }
496
- startListeningRef.current();
497
- },
498
- },
499
- );
500
- }, [commitUtterance, endTurn, fail, levels, setAgent, showDraft]);
501
- startListeningRef.current = startListening;
502
-
503
- // ============== THINKING ==============
504
-
505
- const runTurn = useCallback(
506
- (text: string) => {
507
- // Settle the person's words as a message of their own.
508
- const draftId = draftIdRef.current;
509
- draftIdRef.current = null;
510
- if (draftId !== null) {
511
- patchMessage(draftId, { content: text, isStreaming: false });
512
- } else {
513
- updateMessages((previous) => [
514
- ...previous,
515
- { id: nextId(), role: 'user', content: text, timestamp: Date.now(), isStreaming: false },
516
- ]);
517
- }
518
-
519
- const history: VoiceTurnMessage[] = messagesRef.current
520
- .filter((message) => message.content.trim() !== '' && message.id !== draftId)
521
- .map((message) => ({ role: message.role, content: message.content }))
522
- // The turn itself is `text`, not history.
523
- .slice(0, draftId === null ? -1 : undefined);
524
-
525
- const assistantId = nextId();
526
- updateMessages((previous) => [
527
- ...previous,
528
- {
529
- id: assistantId,
530
- role: 'assistant',
531
- speaker: 'primary',
532
- content: '',
533
- timestamp: Date.now(),
534
- isStreaming: true,
535
- },
536
- ]);
537
-
538
- const turn: ActiveTurn = {
539
- controller: new AbortController(),
540
- assistantId,
541
- consumed: 0,
542
- chunkCount: 0,
543
- text: '',
544
- speaking: '',
545
- clips: [],
546
- closed: false,
547
- wake: null,
548
- synthesisFailed: false,
549
- };
550
- turnRef.current = turn;
551
- setAgent('thinking');
552
- if (mountedRef.current) setTurnErrorCode(null);
553
-
554
- // The microphone stays open through thinking and speaking, for barge-in.
555
- startListeningRef.current();
556
-
557
- const playing = playTurn(turn);
558
- const signal = turn.controller.signal;
559
-
560
- void (async () => {
561
- let lastText = '';
562
- try {
563
- await configRef.current.sendTurn({
564
- text,
565
- history,
566
- signal,
567
- onText: (answerSoFar) => {
568
- if (signal.aborted) return;
569
- lastText = answerSoFar;
570
- turn.text = stripTitleTagsPartial(answerSoFar, { trim: false });
571
- patchMessage(assistantId, { content: turn.text.trim() });
572
- queueChunks(turn, false);
573
- },
574
- });
575
- } catch (caught: unknown) {
576
- if (signal.aborted) return;
577
- console.error('[useVoiceRoom] Turn failed:', caught);
578
- fail('turn-failed', errorMessage(caught, VOICE_ERROR_MESSAGES['turn-failed']));
579
- return;
580
- }
581
- if (signal.aborted) return;
582
-
583
- turn.text = stripTitleTags(lastText, { trim: false });
584
- patchMessage(assistantId, { content: turn.text.trim(), isStreaming: false });
585
- queueChunks(turn, true);
586
- turn.closed = true;
587
- turn.wake?.();
588
- await playing;
589
- if (signal.aborted || turnRef.current !== turn) return;
590
-
591
- turnRef.current = null;
592
- levels.setPlayback(0);
593
- // The session that listened through the answer has heard the answer; a
594
- // fresh one starts the next utterance clean.
595
- sessionRef.current?.abort();
596
- if (turn.text.trim() === '') {
597
- updateMessages((previous) => previous.filter((message) => message.id !== assistantId));
598
- if (mountedRef.current) setTurnErrorCode('no-answer');
599
- } else if (turn.synthesisFailed) {
600
- if (mountedRef.current) setTurnErrorCode('not-played');
601
- }
602
- setAgent('listening');
603
- startListeningRef.current();
604
- })();
605
- },
606
- [fail, levels, patchMessage, playTurn, queueChunks, setAgent, updateMessages],
607
- );
608
- runTurnRef.current = runTurn;
609
-
610
- // ============== CONNECT ==============
611
-
612
118
  const connect = useCallback(async () => {
613
- if (phaseRef.current !== 'off') return;
614
- setCallFailure(null);
615
- setTurnErrorCode(null);
616
- setRoomState('connecting');
617
-
618
- if (!isSpeechRecognitionAvailable()) {
619
- setCallFailure({
620
- code: 'voice-unavailable',
621
- message: VOICE_ERROR_MESSAGES['voice-unavailable'],
622
- });
623
- setRoomState('error');
624
- return;
625
- }
626
- if (getToken() === null) {
627
- setCallFailure({
628
- code: 'not-authenticated',
629
- message: VOICE_ERROR_MESSAGES['not-authenticated'],
630
- });
631
- setRoomState('error');
632
- return;
633
- }
634
-
635
- const refusal = await requestSpeechRecognitionPermission();
636
- if (!mountedRef.current) return;
637
- if (refusal !== null) {
638
- const code = speechFailureCode(refusal) ?? 'microphone-denied';
639
- setCallFailure({ code, message: VOICE_ERROR_MESSAGES[code] });
640
- setRoomState('error');
641
- return;
642
- }
643
-
644
- // The recognizer that knows the call's language, as dictation picks it.
645
- const choice = await chooseSpeechRecognizer(configRef.current.lang);
646
- if (!mountedRef.current) return;
647
- if ('failure' in choice) {
648
- const code = speechFailureCode(choice.failure) ?? 'speech-language';
649
- setCallFailure({ code, message: VOICE_ERROR_MESSAGES[code] });
650
- setRoomState('error');
651
- return;
652
- }
653
- recognizerRef.current = { lang: choice.lang, service: choice.service };
654
-
655
- rapidEndsRef.current = 0;
656
- setRoomState('connected');
657
- setAgent('listening');
658
- startListeningRef.current();
659
- }, [getToken, setAgent]);
660
-
661
- // ============== DISCONNECT ==============
662
-
119
+ const live = callRef.current?.snapshot().state.kind;
120
+ if (live !== undefined && live !== 'ended' && live !== 'error') return;
121
+ const current = optionsRef.current;
122
+ const call = new VoiceCall({
123
+ apiUrl: current.apiUrl || API_URL,
124
+ getToken,
125
+ conversationId: current.conversationId ?? newConversationId(),
126
+ ...(current.agentId === undefined ? {} : { agentId: current.agentId }),
127
+ voice: current.voicePreference ?? 'female',
128
+ locale: current.lang ?? defaultSpeechLanguage(),
129
+ createEngine: current.createAudioEngine ?? createVoiceAudioEngine,
130
+ openSocket: current.openSocket ?? openWebSocket,
131
+ levels,
132
+ onChange: (next) => {
133
+ if (mountedRef.current && callRef.current === call) setSnapshot(next);
134
+ },
135
+ onMessagesSaved: (conversationId, ids) =>
136
+ optionsRef.current.onMessagesSaved?.(conversationId, ids),
137
+ onApprovalRequested: (toolName) => optionsRef.current.onApprovalRequested?.(toolName),
138
+ });
139
+ callRef.current = call;
140
+ await call.start();
141
+ }, [getToken, levels]);
142
+
143
+ /** Hang up and forget the call: the room is back to `disconnected`. */
663
144
  const disconnect = useCallback(() => {
664
- teardown();
665
- draftIdRef.current = null;
666
- messagesRef.current = [];
667
- mutedRef.current = false;
668
- setRoomState('disconnected');
669
- setAgentStateValue('idle');
670
- setCallFailure(null);
671
- setTurnErrorCode(null);
672
- setIsMuted(false);
673
- setMessagesState([]);
674
- setCurrentSpeaker(null);
675
- }, [teardown]);
676
-
677
- // ============== MUTE ==============
145
+ const call = callRef.current;
146
+ callRef.current = null;
147
+ call?.hangUp();
148
+ levels.reset();
149
+ if (mountedRef.current) setSnapshot(IDLE);
150
+ }, [levels]);
678
151
 
679
152
  const toggleMute = useCallback(() => {
680
- if (phaseRef.current === 'off') return;
681
- const next = !mutedRef.current;
682
- mutedRef.current = next;
683
- setIsMuted(next);
684
- if (next) {
685
- // What was half-said is dropped, not sent: muting mid-sentence means "not that".
686
- clearTimers();
687
- committingRef.current = false;
688
- heardRef.current = '';
689
- discardDraft();
690
- sessionRef.current?.abort();
691
- levels.setCapture(0);
692
- } else {
693
- startListeningRef.current();
694
- }
695
- }, [discardDraft, levels]);
696
-
697
- // ============== CLEANUP ON UNMOUNT ==============
153
+ const call = callRef.current;
154
+ if (call === null) return;
155
+ call.setMuted(!call.snapshot().isMuted);
156
+ }, []);
698
157
 
699
158
  useEffect(() => {
700
159
  mountedRef.current = true;
701
160
  return () => {
702
161
  mountedRef.current = false;
703
- teardown();
162
+ const call = callRef.current;
163
+ callRef.current = null;
164
+ call?.hangUp();
704
165
  };
705
- }, [teardown]);
166
+ }, []);
706
167
 
168
+ const { state } = snapshot;
169
+ const roomState = roomStateOf(state);
170
+ const errorCode: VoiceErrorCode | null = state.kind === 'error' ? state.code : null;
171
+ const endedReason: VoiceCloseReason | null = state.kind === 'ended' ? state.reason : null;
707
172
  const room: VoiceLevelSource = levels;
708
173
 
709
174
  return {
710
- /** The call's live levels, for `useAudioLevelMonitor`. Named for the room it replaced. */
175
+ /** The call's live levels, for `useAudioLevelMonitor`. */
711
176
  room,
177
+ /** The honest state machine (ADR 0017 §4). */
178
+ callState: state,
712
179
  roomState,
713
- agentState,
714
- isMuted,
715
- /** Why the call ended, in English. */
716
- error: callFailure?.message ?? null,
180
+ agentState: agentStateOf(state),
181
+ isMuted: snapshot.isMuted,
182
+ /** Why the call failed, in English. */
183
+ error: errorCode === null ? null : VOICE_ERROR_MESSAGES[errorCode],
717
184
  /** The same, as a code an app can translate. */
718
- errorCode: callFailure?.code ?? null,
719
- turnError: turnErrorCode === null ? null : VOICE_ERROR_MESSAGES[turnErrorCode],
720
- turnErrorCode,
721
- messages,
722
- currentSpeaker,
185
+ errorCode,
186
+ /** Why the call ended without failing; `call-moved-elsewhere` is one. */
187
+ endedReason,
188
+ /** Milliseconds left once the server warned the call is about to end. */
189
+ endingSoonMs: snapshot.endingSoonMs,
190
+ /** The model serving the call (its name may be shown; the operator never is). */
191
+ model: snapshot.model,
192
+ /** The live transcript, both sides. The stored one is the conversation's. */
193
+ messages: snapshot.messages as VoiceMessage[],
194
+ currentSpeaker:
195
+ state.kind === 'speaking'
196
+ ? ('primary' as const)
197
+ : state.kind === 'user-speaking'
198
+ ? ('user' as const)
199
+ : null,
723
200
  connect,
724
201
  disconnect,
725
202
  toggleMute,