@alia.onl/sdk 13.2.0 → 15.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/CHANGELOG.md +129 -0
  2. package/README.md +52 -20
  3. package/package.json +8 -4
  4. package/src/components/AliaAvatarIcon.tsx +30 -0
  5. package/src/components/AliaChatContent.tsx +4 -7
  6. package/src/components/AliaChatMessageList.tsx +2 -10
  7. package/src/components/AliaChatScreen.tsx +1 -1
  8. package/src/components/AliaChatSheet.tsx +15 -4
  9. package/src/components/AliaComposer.tsx +57 -11
  10. package/src/components/voice/VoiceControls.tsx +28 -4
  11. package/src/components/voice/VoiceSession.tsx +23 -7
  12. package/src/hooks/useAliaActivity.ts +31 -0
  13. package/src/hooks/useAliaChat.ts +14 -3
  14. package/src/hooks/useAmbientWave.ts +3 -3
  15. package/src/hooks/useAudioLevelMonitor.ts +2 -2
  16. package/src/hooks/useDictation.ts +295 -0
  17. package/src/hooks/useTTS.ts +24 -63
  18. package/src/hooks/useVoiceRoom.ts +148 -736
  19. package/src/index.ts +9 -3
  20. package/src/lib/activity-socket.ts +102 -0
  21. package/src/lib/activity-store.ts +53 -0
  22. package/src/lib/api-url.ts +2 -0
  23. package/src/lib/audio-level.ts +2 -2
  24. package/src/lib/dictation-recorder-types.ts +37 -0
  25. package/src/lib/dictation-recorder.native.ts +90 -0
  26. package/src/lib/dictation-recorder.ts +131 -0
  27. package/src/lib/dictation.ts +208 -0
  28. package/src/lib/labels.tsx +3 -0
  29. package/src/lib/pcm.ts +57 -0
  30. package/src/lib/speech-language.ts +1 -1
  31. package/src/lib/speech-messages.ts +43 -97
  32. package/src/lib/speech-synthesis.ts +5 -1
  33. package/src/lib/turn-state.ts +15 -0
  34. package/src/lib/voice-audio-engine-types.ts +59 -0
  35. package/src/lib/voice-audio-engine.native.ts +232 -0
  36. package/src/lib/voice-audio-engine.ts +350 -0
  37. package/src/lib/voice-call.ts +539 -0
  38. package/src/lib/voice-levels.ts +7 -3
  39. package/src/lib/voice-protocol.ts +309 -0
  40. package/src/types.ts +49 -4
  41. package/src/voice.ts +31 -12
  42. package/tsconfig.json +2 -0
  43. package/src/hooks/useSpeechToText.ts +0 -268
  44. package/src/lib/speech-recognition-types.ts +0 -108
  45. package/src/lib/speech-recognition.native.ts +0 -237
  46. package/src/lib/speech-recognition.ts +0 -308
  47. package/src/lib/voice-text.ts +0 -198
  48. package/src/lib/voice-turn.ts +0 -89
@@ -1,790 +1,202 @@
1
1
  /**
2
- * A voice conversation, run on the device as a turn loop.
2
+ * A voice call: speech to speech through Alia's `/v1/voice/session`
3
+ * (ADR 0017).
3
4
  *
4
- * listening ──(the person stops talking)──▶ thinking ──(first sentence)──▶ speaking
5
- * ▲ │ │
6
- * └──────────(answer spoken, or the person talks over it)──────────────────┘
5
+ * Alia holds the realtime session with Kaana — instructions, memory, tools
6
+ * (run in Alia, never here), the transcript stored in the conversation, credits
7
+ * and typed errors. Turn detection and barge-in are the model's, on the server.
8
+ * This hook captures and plays PCM through the platform's audio engine
9
+ * (`lib/voice-audio-engine*`) and exposes a state it can prove (`callState`,
10
+ * see `VoiceCallState`), with `roomState` / `agentState` as the coarse
11
+ * projections the call bar, the ambient field and the sound effects read.
7
12
  *
8
- * - **Listening** is on-device speech recognition (`lib/speech-recognition*`).
9
- * An utterance ends after `endOfUtteranceMs` without a new word.
10
- * - **Thinking** sends the utterance through the ordinary chat path — a
11
- * `VoiceTurnSender`, by default `POST /v1/chat/completions` marked
12
- * `responseMode: 'voice'` — and streams the answer into the transcript.
13
- * - **Speaking** synthesizes the answer through `POST /v1/audio/speech`
14
- * sentence by sentence as it streams, so the first sentence plays while the
15
- * rest is still being written, and the next clip is fetched while the
16
- * current one plays.
17
- * - **Barge-in:** the microphone stays open while Alia thinks and speaks. When
18
- * the person says a few words that are not the answer echoing back, the turn
19
- * is aborted — playback stops, the chat request is cancelled — and what they
20
- * are saying becomes the next utterance.
21
- * - **A voice that cannot speak ends the call.** When a clip cannot be made
22
- * (`synthesisError`, typed from the route's code) or played, the turn stops
23
- * speaking, its answer finishes streaming into the transcript, and the call
24
- * ends with the reason as `errorCode` — never a call that listens on, mute.
25
- *
26
- * It replaced a LiveKit room (`POST /v1/voice/token`) whose realtime model and
27
- * transcription called providers directly; Alia's only inference path is
28
- * Alia → Oxy → Kaana, which serves chat and speech but no realtime session or
29
- * transcription. The public shape — `connect` / `disconnect`, `roomState`,
30
- * `agentState`, `messages`, `toggleMute` — is unchanged, so its consumers did
31
- * not change with it.
13
+ * Nothing is recognized on the device: there is no browser or OS speech
14
+ * recognizer anywhere in a call.
32
15
  */
33
16
 
34
- import { useState, useCallback, useRef, useEffect, useMemo } from 'react';
17
+ import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
35
18
  import { useOxy } from '@oxy.so/services/ui/client';
36
- import { errorMessage } from '../lib/utils';
37
- import type { RoomState, AgentState, VoiceMessage } from '../types';
38
- import {
39
- isSpeechRecognitionAvailable,
40
- chooseSpeechRecognizer,
41
- requestSpeechRecognitionPermission,
42
- startSpeechRecognition,
43
- } from '../lib/speech-recognition';
44
- import type {
45
- SpeechRecognitionFailure,
46
- SpeechRecognitionSession,
47
- } from '../lib/speech-recognition-types';
48
- import {
49
- speechFailureCode,
50
- synthesisFailureCode,
51
- VOICE_ERROR_MESSAGES,
52
- type VoiceErrorCode,
53
- } from '../lib/speech-messages';
19
+ import type { AgentState, RoomState, VoiceCallState, VoiceMessage } from '../types';
20
+ import { VOICE_ERROR_MESSAGES, type VoiceErrorCode } from '../lib/speech-messages';
54
21
  import { defaultSpeechLanguage } from '../lib/speech-language';
55
- import {
56
- playSpeechClip,
57
- requestSpeechClip,
58
- SpeechClipError,
59
- type ProductVoice,
60
- } from '../lib/speech-synthesis';
22
+ import { createVoiceAudioEngine } from '../lib/voice-audio-engine';
23
+ import type { VoiceAudioEngine } from '../lib/voice-audio-engine-types';
24
+ import { VoiceCall, type VoiceCallSnapshot, type VoiceSocketFactory } from '../lib/voice-call';
61
25
  import { VoiceLevelChannel, type VoiceLevelSource } from '../lib/voice-levels';
62
- import {
63
- createAliaVoiceTurnSender,
64
- type VoiceTurnMessage,
65
- type VoiceTurnSender,
66
- } from '../lib/voice-turn';
67
- import {
68
- dropEchoPrefix,
69
- isInterruption,
70
- stripTitleTags,
71
- stripTitleTagsPartial,
72
- takeSpeechChunks,
73
- } from '../lib/voice-text';
26
+ import type { ProductVoice, VoiceCloseReason } from '../lib/voice-protocol';
74
27
 
75
28
  const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
76
29
 
77
- // ============== OPTIONS ==============
78
-
79
30
  export interface UseVoiceRoomOptions {
80
31
  apiUrl?: string;
81
- /** Which of the product's two voices answers. Sent to `/v1/audio/speech` as is. */
82
- voicePreference?: ProductVoice;
83
- /** Bearer for speech synthesis; defaults to the surrounding Oxy session's. */
84
- accessToken?: string;
85
- /**
86
- * Speech model for the answers. Omitted, the request carries no `model` and
87
- * the server's own speech model answers.
88
- */
89
- model?: string;
90
32
  /**
91
- * Chat model (`publisher/model`) the default sender asks for; omitted, the
92
- * server's default. Ignored when `sendTurn`
93
- * is given — that sender carries its own conversation's choice.
94
- */
95
- chatModel?: string;
96
- /**
97
- * The agent this call belongs to, when it is a thread with one.
98
- *
99
- * Without it the call is ordinary Alia. Only the default sender reads it; a
100
- * custom `sendTurn` is already bound to its conversation's agent.
33
+ * The conversation the call is stored in. Omitted, every call starts a
34
+ * conversation of its own (a fresh id per `connect`).
101
35
  */
36
+ conversationId?: string;
37
+ /** The agent this call talks to; omitted, Alia. */
102
38
  agentId?: string;
103
- /** Language to recognize (BCP-47). Defaults to the platform's. */
39
+ /** Which of the product's two voices answers. */
40
+ voicePreference?: ProductVoice;
41
+ /** Bearer for the call; defaults to the surrounding Oxy session's. */
42
+ accessToken?: string;
43
+ /** Language spoken (BCP-47). Defaults to the platform's. */
104
44
  lang?: string;
105
- /**
106
- * How each turn reaches Alia. Defaults to `createAliaVoiceTurnSender`. The
107
- * Alia app passes its conversation's own send, so a call's turns are that
108
- * conversation's turns — persisted and shown like typed ones.
109
- */
110
- sendTurn?: VoiceTurnSender;
111
- /** Silence, in ms, that ends an utterance. */
112
- endOfUtteranceMs?: number;
113
- /** Let the person interrupt by talking. On by default. */
114
- bargeIn?: boolean;
45
+ /** Alia stored messages of the call: refresh the thread. */
46
+ onMessagesSaved?: (conversationId: string, messageIds: readonly string[]) => void;
47
+ /** A tool filed an action for the person's OK; its card is in the conversation. */
48
+ onApprovalRequested?: (toolName: string) => void;
49
+ /** Test seams: the device's audio and the socket. */
50
+ createAudioEngine?: () => VoiceAudioEngine;
51
+ openSocket?: VoiceSocketFactory;
115
52
  }
116
53
 
117
- // ============== TUNING ==============
118
-
119
- const DEFAULT_END_OF_UTTERANCE_MS = 1_100;
120
- /** How long a requested stop may take to deliver the final words. */
121
- const COMMIT_TIMEOUT_MS = 1_500;
122
- /** Words needed before talking over the answer counts as an interruption. */
123
- const BARGE_IN_MIN_WORDS = 2;
124
- /** A recognizer that keeps ending within this of starting is failing, not idling. */
125
- const RAPID_END_MS = 400;
126
- const MAX_RAPID_ENDS = 5;
54
+ const IDLE: VoiceCallSnapshot = {
55
+ state: { kind: 'idle' },
56
+ messages: [],
57
+ isMuted: false,
58
+ endingSoonMs: null,
59
+ model: null,
60
+ };
127
61
 
128
- // ============== INTERNAL STATE ==============
62
+ const openWebSocket: VoiceSocketFactory = (url) =>
63
+ new WebSocket(url) as unknown as ReturnType<VoiceSocketFactory>;
129
64
 
130
- type Phase = 'off' | 'listening' | 'thinking' | 'speaking';
65
+ function newConversationId(): string {
66
+ const random = globalThis.crypto?.randomUUID?.();
67
+ return random ?? `voice-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
68
+ }
131
69
 
132
- interface ActiveTurn {
133
- readonly controller: AbortController;
134
- readonly assistantId: string;
135
- /** Characters of the answer already handed to the speech queue. */
136
- consumed: number;
137
- chunkCount: number;
138
- /** The answer so far, title tags stripped and whitespace kept (see `stripTitleTags`). */
139
- text: string;
140
- /** Everything this turn has said aloud so far, for telling echo from the person. */
141
- speaking: string;
142
- readonly clips: Array<{ readonly text: string; clip: Promise<string | null> | null }>;
143
- closed: boolean;
144
- /** Wakes the player when a clip is queued or the answer ends. */
145
- wake: (() => void) | null;
146
- /**
147
- * Why this turn stopped speaking: a clip that could not be made (`synthesis`
148
- * holds what the route said) or one the device could not play. The call ends
149
- * with `code` once the answer has finished streaming.
150
- */
151
- speechFailure: {
152
- readonly code: VoiceErrorCode;
153
- readonly synthesis: SpeechClipError | null;
154
- } | null;
70
+ /** The coarse connection state the bar and the wave read. */
71
+ export function roomStateOf(state: VoiceCallState): RoomState {
72
+ switch (state.kind) {
73
+ case 'idle':
74
+ case 'ended':
75
+ return 'disconnected';
76
+ case 'connecting':
77
+ return 'connecting';
78
+ case 'reconnecting':
79
+ return 'reconnecting';
80
+ case 'error':
81
+ return 'error';
82
+ default:
83
+ return 'connected';
84
+ }
155
85
  }
156
86
 
157
- // ============== HOOK ==============
87
+ /** Who has the turn, coarsely. */
88
+ export function agentStateOf(state: VoiceCallState): AgentState {
89
+ switch (state.kind) {
90
+ case 'listening':
91
+ case 'user-speaking':
92
+ return 'listening';
93
+ case 'thinking':
94
+ case 'using-tool':
95
+ return 'thinking';
96
+ case 'speaking':
97
+ return 'speaking';
98
+ default:
99
+ return 'idle';
100
+ }
101
+ }
158
102
 
159
103
  export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
160
- const apiUrl = options.apiUrl || API_URL;
161
- const voicePref: ProductVoice = options.voicePreference ?? 'female';
162
- const speechModel = options.model;
163
- const endOfUtteranceMs = options.endOfUtteranceMs ?? DEFAULT_END_OF_UTTERANCE_MS;
164
- const bargeIn = options.bargeIn ?? true;
165
-
166
- const [roomState, setRoomState] = useState<RoomState>('disconnected');
167
- const [agentState, setAgentStateValue] = useState<AgentState>('idle');
168
- const [isMuted, setIsMuted] = useState(false);
169
- /**
170
- * Why the call ended, as a code and as the sentence to show. The sentence is
171
- * the code's English copy, except for `turn-failed`, which carries what the
172
- * turn sender threw.
173
- */
174
- const [callFailure, setCallFailure] = useState<{ code: VoiceErrorCode; message: string } | null>(
175
- null,
176
- );
177
- /** A turn that failed without ending the call: no answer came back. */
178
- const [turnErrorCode, setTurnErrorCode] = useState<VoiceErrorCode | null>(null);
179
- /** The clip that could not be made, when that is what ended the call. */
180
- const [synthesisError, setSynthesisError] = useState<SpeechClipError | null>(null);
181
- const [messages, setMessagesState] = useState<VoiceMessage[]>([]);
182
- const [currentSpeaker, setCurrentSpeaker] = useState<'primary' | 'user' | null>(null);
183
-
184
104
  const { oxyServices } = useOxy();
185
-
186
105
  const levels = useMemo(() => new VoiceLevelChannel(), []);
187
-
188
- // Everything the loop reads from callbacks lives in refs: recognizer events
189
- // arrive outside React, and a value closed over at `connect` would be stale.
190
- const phaseRef = useRef<Phase>('off');
191
- const mutedRef = useRef(false);
106
+ const [snapshot, setSnapshot] = useState<VoiceCallSnapshot>(IDLE);
107
+ const callRef = useRef<VoiceCall | null>(null);
192
108
  const mountedRef = useRef(true);
193
- const messagesRef = useRef<VoiceMessage[]>([]);
194
- const sessionRef = useRef<SpeechRecognitionSession | null>(null);
195
- /** The recognizer and tag the call listens with, chosen when it connects. */
196
- const recognizerRef = useRef<{ lang: string; service?: string }>({ lang: '' });
197
- const heardRef = useRef('');
198
- const committingRef = useRef(false);
199
- const silenceTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
200
- const commitTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
201
- const turnRef = useRef<ActiveTurn | null>(null);
202
- const draftIdRef = useRef<string | null>(null);
203
- /** What the call had said when the person cut in; this session's echo of it is not theirs. */
204
- const echoContextRef = useRef('');
205
- const sessionStartedAtRef = useRef(0);
206
- const rapidEndsRef = useRef(0);
207
- const sessionFailedRef = useRef<SpeechRecognitionFailure | null>(null);
208
- const msgIdRef = useRef(0);
209
- const sessionPrefixRef = useRef(`vm-${Date.now().toString(36)}`);
210
-
211
- const config = {
212
- apiUrl,
213
- voicePref,
214
- speechModel,
215
- endOfUtteranceMs,
216
- bargeIn,
217
- lang: options.lang ?? defaultSpeechLanguage(),
218
- accessToken: options.accessToken,
219
- sendTurn:
220
- options.sendTurn ??
221
- createAliaVoiceTurnSender({
222
- oxyServices,
223
- apiUrl,
224
- ...(options.chatModel === undefined ? {} : { model: options.chatModel }),
225
- ...(options.agentId === undefined ? {} : { agentId: options.agentId }),
226
- }),
227
- };
228
- const configRef = useRef(config);
229
- configRef.current = config;
230
-
231
- // ============== STATE HELPERS ==============
232
-
233
- const setAgent = useCallback((phase: Phase) => {
234
- phaseRef.current = phase;
235
- if (!mountedRef.current) return;
236
- setAgentStateValue(phase === 'off' ? 'idle' : phase);
237
- setCurrentSpeaker(phase === 'thinking' || phase === 'speaking' ? 'primary' : null);
238
- }, []);
239
-
240
- const updateMessages = useCallback((update: (previous: VoiceMessage[]) => VoiceMessage[]) => {
241
- messagesRef.current = update(messagesRef.current);
242
- if (mountedRef.current) setMessagesState(messagesRef.current);
243
- }, []);
244
-
245
- const nextId = (): string => {
246
- msgIdRef.current += 1;
247
- return `${sessionPrefixRef.current}-${msgIdRef.current}`;
248
- };
249
-
250
- const patchMessage = useCallback(
251
- (id: string, patch: Partial<VoiceMessage>) => {
252
- updateMessages((previous) =>
253
- previous.map((message) => (message.id === id ? { ...message, ...patch } : message)),
254
- );
255
- },
256
- [updateMessages],
257
- );
109
+ const optionsRef = useRef(options);
110
+ optionsRef.current = options;
258
111
 
259
112
  const getToken = useCallback((): string | null => {
260
- const { accessToken } = configRef.current;
113
+ const { accessToken } = optionsRef.current;
261
114
  if (accessToken) return accessToken;
262
- return oxyServices.session.accessToken;
115
+ return oxyServices.session.accessToken ?? null;
263
116
  }, [oxyServices]);
264
117
 
265
- const clearTimers = (): void => {
266
- if (silenceTimerRef.current !== null) clearTimeout(silenceTimerRef.current);
267
- if (commitTimerRef.current !== null) clearTimeout(commitTimerRef.current);
268
- silenceTimerRef.current = null;
269
- commitTimerRef.current = null;
270
- };
271
-
272
- // ============== TEARDOWN ==============
273
-
274
- const endTurn = useCallback(
275
- (turn: ActiveTurn) => {
276
- turn.controller.abort();
277
- turn.closed = true;
278
- turn.wake?.();
279
- if (turnRef.current === turn) turnRef.current = null;
280
- const message = messagesRef.current.find((candidate) => candidate.id === turn.assistantId);
281
- if (message?.isStreaming) {
282
- if (message.content.trim() === '') {
283
- updateMessages((previous) =>
284
- previous.filter((candidate) => candidate.id !== turn.assistantId),
285
- );
286
- } else {
287
- patchMessage(turn.assistantId, { isStreaming: false });
288
- }
289
- }
290
- levels.setPlayback(0);
291
- },
292
- [levels, patchMessage, updateMessages],
293
- );
294
-
295
- const discardDraft = useCallback(() => {
296
- const draftId = draftIdRef.current;
297
- draftIdRef.current = null;
298
- if (draftId !== null)
299
- updateMessages((previous) => previous.filter((message) => message.id !== draftId));
300
- }, [updateMessages]);
301
-
302
- /** Stop everything; the caller decides what state the room is left in. */
303
- const teardown = useCallback(() => {
304
- phaseRef.current = 'off';
305
- clearTimers();
306
- committingRef.current = false;
307
- const turn = turnRef.current;
308
- if (turn !== null) endTurn(turn);
309
- sessionRef.current?.abort();
310
- sessionRef.current = null;
311
- heardRef.current = '';
312
- levels.reset();
313
- }, [endTurn, levels]);
314
-
315
- const fail = useCallback(
316
- (code: VoiceErrorCode, message: string = VOICE_ERROR_MESSAGES[code]) => {
317
- teardown();
318
- discardDraft();
319
- if (!mountedRef.current) return;
320
- setCallFailure({ code, message });
321
- setRoomState('error');
322
- setAgentStateValue('idle');
323
- setCurrentSpeaker(null);
324
- },
325
- [discardDraft, teardown],
326
- );
327
-
328
- // ============== SPEAKING ==============
329
-
330
- /** The first reason a turn could not speak is the one the call ends with. */
331
- const stopSpeaking = useCallback(
332
- (turn: ActiveTurn, code: VoiceErrorCode, synthesis: SpeechClipError | null) => {
333
- turn.speechFailure ??= { code, synthesis };
334
- },
335
- [],
336
- );
337
-
338
- const synthesize = useCallback(
339
- (turn: ActiveTurn, text: string): Promise<string | null> => {
340
- const token = getToken();
341
- if (token === null) {
342
- const failure = new SpeechClipError(VOICE_ERROR_MESSAGES['not-authenticated'], {
343
- status: null,
344
- code: 'NOT_AUTHENTICATED',
345
- });
346
- stopSpeaking(turn, synthesisFailureCode(failure), failure);
347
- return Promise.resolve(null);
348
- }
349
- const { apiUrl: url, speechModel: model, voicePref: voice } = configRef.current;
350
- return requestSpeechClip({
351
- apiUrl: url,
352
- token,
353
- model,
354
- input: text,
355
- voice,
356
- signal: turn.controller.signal,
357
- }).catch((caught: unknown) => {
358
- if (turn.controller.signal.aborted) return null;
359
- const failure =
360
- caught instanceof SpeechClipError
361
- ? caught
362
- : new SpeechClipError(errorMessage(caught, 'TTS failed'), { status: null, code: null });
363
- console.error('[useVoiceRoom] Speech synthesis failed:', {
364
- status: failure.status,
365
- code: failure.code,
366
- requestId: failure.requestId,
367
- });
368
- stopSpeaking(turn, synthesisFailureCode(failure), failure);
369
- return null;
370
- });
371
- },
372
- [getToken, stopSpeaking],
373
- );
374
-
375
- /**
376
- * Play the turn's clips in order as they arrive, fetching one ahead. Returns
377
- * early, without skipping to a later sentence, once a clip cannot be made or
378
- * played.
379
- */
380
- const playTurn = useCallback(
381
- async (turn: ActiveTurn): Promise<void> => {
382
- const signal = turn.controller.signal;
383
- for (let index = 0; ; index += 1) {
384
- while (index >= turn.clips.length && !turn.closed) {
385
- await new Promise<void>((resolve) => {
386
- turn.wake = resolve;
387
- });
388
- turn.wake = null;
389
- }
390
- if (signal.aborted || index >= turn.clips.length) return;
391
- const item = turn.clips[index];
392
- item.clip ??= synthesize(turn, item.text);
393
- const next = turn.clips[index + 1];
394
- if (next !== undefined) next.clip ??= synthesize(turn, next.text);
395
-
396
- const uri = await item.clip;
397
- if (signal.aborted) return;
398
- if (uri === null) {
399
- setAgent('thinking');
400
- levels.setPlayback(0);
401
- return;
402
- }
403
- turn.speaking = `${turn.speaking} ${item.text}`;
404
- setAgent('speaking');
405
- try {
406
- await playSpeechClip(uri, { signal, onLevel: (level) => levels.setPlayback(level) });
407
- } catch (caught: unknown) {
408
- if (signal.aborted) return;
409
- console.error('[useVoiceRoom] Playback failed:', caught);
410
- stopSpeaking(turn, 'not-played', null);
411
- setAgent('thinking');
412
- return;
413
- }
414
- }
415
- },
416
- [levels, setAgent, stopSpeaking, synthesize],
417
- );
418
-
419
- const queueChunks = useCallback((turn: ActiveTurn, final: boolean) => {
420
- const pending = turn.text.slice(turn.consumed);
421
- const { chunks, rest } = takeSpeechChunks(pending, { final, isFirst: turn.chunkCount === 0 });
422
- turn.consumed = turn.text.length - rest.length;
423
- for (const text of chunks) {
424
- turn.chunkCount += 1;
425
- // The next clip is fetched now if the player is waiting on it; the
426
- // player fetches the rest one ahead of what it is playing.
427
- turn.clips.push({ text, clip: null });
428
- }
429
- if (chunks.length > 0) turn.wake?.();
430
- }, []);
431
-
432
- // ============== LISTENING ==============
433
-
434
- // Declared through refs so the recognizer callbacks and the turn runner can
435
- // call each other without a cycle in their dependency lists.
436
- const startListeningRef = useRef<() => void>(() => undefined);
437
- const runTurnRef = useRef<(text: string) => void>(() => undefined);
438
-
439
- const commitUtterance = useCallback(() => {
440
- const session = sessionRef.current;
441
- if (session === null || committingRef.current) return;
442
- committingRef.current = true;
443
- session.stop();
444
- commitTimerRef.current = setTimeout(() => {
445
- commitTimerRef.current = null;
446
- sessionRef.current?.abort();
447
- }, COMMIT_TIMEOUT_MS);
448
- }, []);
449
-
450
- const showDraft = useCallback(
451
- (transcript: string) => {
452
- const draftId = draftIdRef.current;
453
- if (draftId !== null) {
454
- patchMessage(draftId, { content: transcript });
455
- return;
456
- }
457
- const id = nextId();
458
- draftIdRef.current = id;
459
- updateMessages((previous) => [
460
- ...previous,
461
- { id, role: 'user', content: transcript, timestamp: Date.now(), isStreaming: true },
462
- ]);
463
- if (mountedRef.current) setCurrentSpeaker('user');
464
- },
465
- [patchMessage, updateMessages],
466
- );
467
-
468
- const startListening = useCallback(() => {
469
- if (phaseRef.current === 'off' || mutedRef.current || sessionRef.current !== null) return;
470
- heardRef.current = '';
471
- committingRef.current = false;
472
- sessionFailedRef.current = null;
473
- echoContextRef.current = '';
474
- sessionStartedAtRef.current = Date.now();
475
-
476
- sessionRef.current = startSpeechRecognition(
477
- { ...recognizerRef.current, echoCancellation: true },
478
- {
479
- onResult: ({ transcript }) => {
480
- if (phaseRef.current === 'off' || committingRef.current) return;
481
-
482
- const turn = turnRef.current;
483
- if (turn !== null) {
484
- // Alia is thinking or speaking: this is the person cutting in, or
485
- // the answer coming back through the microphone.
486
- heardRef.current = '';
487
- if (!configRef.current.bargeIn) return;
488
- if (!isInterruption(transcript, turn.speaking, BARGE_IN_MIN_WORDS)) return;
489
- echoContextRef.current = turn.speaking;
490
- endTurn(turn);
491
- setAgent('listening');
492
- }
493
-
494
- const utterance = dropEchoPrefix(transcript, echoContextRef.current);
495
- heardRef.current = utterance;
496
- if (utterance === '') return;
497
- showDraft(utterance);
498
- if (silenceTimerRef.current !== null) clearTimeout(silenceTimerRef.current);
499
- silenceTimerRef.current = setTimeout(() => {
500
- silenceTimerRef.current = null;
501
- commitUtterance();
502
- }, configRef.current.endOfUtteranceMs);
503
- },
504
- onLevel: (level) => {
505
- levels.setCapture(mutedRef.current ? 0 : level);
506
- },
507
- onError: (failure) => {
508
- if (failure.code === 'no-speech' || failure.code === 'aborted') return;
509
- sessionFailedRef.current = failure;
510
- },
511
- onEnd: () => {
512
- sessionRef.current = null;
513
- clearTimers();
514
- const committing = committingRef.current;
515
- committingRef.current = false;
516
- const heard = heardRef.current.trim();
517
- heardRef.current = '';
518
- if (phaseRef.current === 'off') return;
519
-
520
- const failure = sessionFailedRef.current;
521
- if (failure !== null && failure.code !== 'other') {
522
- fail(speechFailureCode(failure) ?? 'speech-failed');
523
- return;
524
- }
525
-
526
- // An utterance in progress when the engine ended is still an utterance.
527
- if (
528
- (committing || draftIdRef.current !== null) &&
529
- heard !== '' &&
530
- turnRef.current === null
531
- ) {
532
- rapidEndsRef.current = 0;
533
- runTurnRef.current(heard);
534
- return;
535
- }
536
-
537
- if (Date.now() - sessionStartedAtRef.current < RAPID_END_MS) {
538
- rapidEndsRef.current += 1;
539
- if (rapidEndsRef.current >= MAX_RAPID_ENDS) {
540
- fail(speechFailureCode(failure ?? { code: 'other' }) ?? 'speech-failed');
541
- return;
542
- }
543
- } else {
544
- rapidEndsRef.current = 0;
545
- }
546
- startListeningRef.current();
547
- },
548
- },
549
- );
550
- }, [commitUtterance, endTurn, fail, levels, setAgent, showDraft]);
551
- startListeningRef.current = startListening;
552
-
553
- // ============== THINKING ==============
554
-
555
- const runTurn = useCallback(
556
- (text: string) => {
557
- // Settle the person's words as a message of their own.
558
- const draftId = draftIdRef.current;
559
- draftIdRef.current = null;
560
- if (draftId !== null) {
561
- patchMessage(draftId, { content: text, isStreaming: false });
562
- } else {
563
- updateMessages((previous) => [
564
- ...previous,
565
- { id: nextId(), role: 'user', content: text, timestamp: Date.now(), isStreaming: false },
566
- ]);
567
- }
568
-
569
- const history: VoiceTurnMessage[] = messagesRef.current
570
- .filter((message) => message.content.trim() !== '' && message.id !== draftId)
571
- .map((message) => ({ role: message.role, content: message.content }))
572
- // The turn itself is `text`, not history.
573
- .slice(0, draftId === null ? -1 : undefined);
574
-
575
- const assistantId = nextId();
576
- updateMessages((previous) => [
577
- ...previous,
578
- {
579
- id: assistantId,
580
- role: 'assistant',
581
- speaker: 'primary',
582
- content: '',
583
- timestamp: Date.now(),
584
- isStreaming: true,
585
- },
586
- ]);
587
-
588
- const turn: ActiveTurn = {
589
- controller: new AbortController(),
590
- assistantId,
591
- consumed: 0,
592
- chunkCount: 0,
593
- text: '',
594
- speaking: '',
595
- clips: [],
596
- closed: false,
597
- wake: null,
598
- speechFailure: null,
599
- };
600
- turnRef.current = turn;
601
- setAgent('thinking');
602
- if (mountedRef.current) setTurnErrorCode(null);
603
-
604
- // The microphone stays open through thinking and speaking, for barge-in.
605
- startListeningRef.current();
606
-
607
- const playing = playTurn(turn);
608
- const signal = turn.controller.signal;
609
-
610
- void (async () => {
611
- let lastText = '';
612
- try {
613
- await configRef.current.sendTurn({
614
- text,
615
- history,
616
- signal,
617
- onText: (answerSoFar) => {
618
- if (signal.aborted) return;
619
- lastText = answerSoFar;
620
- turn.text = stripTitleTagsPartial(answerSoFar, { trim: false });
621
- patchMessage(assistantId, { content: turn.text.trim() });
622
- queueChunks(turn, false);
623
- },
624
- });
625
- } catch (caught: unknown) {
626
- if (signal.aborted) return;
627
- console.error('[useVoiceRoom] Turn failed:', caught);
628
- fail('turn-failed', errorMessage(caught, VOICE_ERROR_MESSAGES['turn-failed']));
629
- return;
630
- }
631
- if (signal.aborted) return;
632
-
633
- turn.text = stripTitleTags(lastText, { trim: false });
634
- patchMessage(assistantId, { content: turn.text.trim(), isStreaming: false });
635
- queueChunks(turn, true);
636
- turn.closed = true;
637
- turn.wake?.();
638
- await playing;
639
- if (signal.aborted || turnRef.current !== turn) return;
640
-
641
- // The answer is in the transcript; a call that cannot say it ends here,
642
- // rather than listening on in silence.
643
- const speechFailure = turn.speechFailure;
644
- if (speechFailure !== null) {
645
- if (mountedRef.current) setSynthesisError(speechFailure.synthesis);
646
- fail(speechFailure.code);
647
- return;
648
- }
649
-
650
- turnRef.current = null;
651
- levels.setPlayback(0);
652
- // The session that listened through the answer has heard the answer; a
653
- // fresh one starts the next utterance clean.
654
- sessionRef.current?.abort();
655
- if (turn.text.trim() === '') {
656
- updateMessages((previous) => previous.filter((message) => message.id !== assistantId));
657
- if (mountedRef.current) setTurnErrorCode('no-answer');
658
- }
659
- setAgent('listening');
660
- startListeningRef.current();
661
- })();
662
- },
663
- [fail, levels, patchMessage, playTurn, queueChunks, setAgent, updateMessages],
664
- );
665
- runTurnRef.current = runTurn;
666
-
667
- // ============== CONNECT ==============
668
-
669
118
  const connect = useCallback(async () => {
670
- if (phaseRef.current !== 'off') return;
671
- setCallFailure(null);
672
- setTurnErrorCode(null);
673
- setSynthesisError(null);
674
- setRoomState('connecting');
675
-
676
- if (!isSpeechRecognitionAvailable()) {
677
- setCallFailure({
678
- code: 'voice-unavailable',
679
- message: VOICE_ERROR_MESSAGES['voice-unavailable'],
680
- });
681
- setRoomState('error');
682
- return;
683
- }
684
- if (getToken() === null) {
685
- setCallFailure({
686
- code: 'not-authenticated',
687
- message: VOICE_ERROR_MESSAGES['not-authenticated'],
688
- });
689
- setRoomState('error');
690
- return;
691
- }
692
-
693
- const refusal = await requestSpeechRecognitionPermission();
694
- if (!mountedRef.current) return;
695
- if (refusal !== null) {
696
- const code = speechFailureCode(refusal) ?? 'microphone-denied';
697
- setCallFailure({ code, message: VOICE_ERROR_MESSAGES[code] });
698
- setRoomState('error');
699
- return;
700
- }
701
-
702
- // The recognizer that knows the call's language, as dictation picks it.
703
- const choice = await chooseSpeechRecognizer(configRef.current.lang);
704
- if (!mountedRef.current) return;
705
- if ('failure' in choice) {
706
- const code = speechFailureCode(choice.failure) ?? 'speech-language';
707
- setCallFailure({ code, message: VOICE_ERROR_MESSAGES[code] });
708
- setRoomState('error');
709
- return;
710
- }
711
- recognizerRef.current = { lang: choice.lang, service: choice.service };
712
-
713
- rapidEndsRef.current = 0;
714
- setRoomState('connected');
715
- setAgent('listening');
716
- startListeningRef.current();
717
- }, [getToken, setAgent]);
718
-
719
- // ============== DISCONNECT ==============
720
-
119
+ const live = callRef.current?.snapshot().state.kind;
120
+ if (live !== undefined && live !== 'ended' && live !== 'error') return;
121
+ const current = optionsRef.current;
122
+ const call = new VoiceCall({
123
+ apiUrl: current.apiUrl || API_URL,
124
+ getToken,
125
+ conversationId: current.conversationId ?? newConversationId(),
126
+ ...(current.agentId === undefined ? {} : { agentId: current.agentId }),
127
+ voice: current.voicePreference ?? 'female',
128
+ locale: current.lang ?? defaultSpeechLanguage(),
129
+ createEngine: current.createAudioEngine ?? createVoiceAudioEngine,
130
+ openSocket: current.openSocket ?? openWebSocket,
131
+ levels,
132
+ onChange: (next) => {
133
+ if (mountedRef.current && callRef.current === call) setSnapshot(next);
134
+ },
135
+ onMessagesSaved: (conversationId, ids) =>
136
+ optionsRef.current.onMessagesSaved?.(conversationId, ids),
137
+ onApprovalRequested: (toolName) => optionsRef.current.onApprovalRequested?.(toolName),
138
+ });
139
+ callRef.current = call;
140
+ await call.start();
141
+ }, [getToken, levels]);
142
+
143
+ /** Hang up and forget the call: the room is back to `disconnected`. */
721
144
  const disconnect = useCallback(() => {
722
- teardown();
723
- draftIdRef.current = null;
724
- messagesRef.current = [];
725
- mutedRef.current = false;
726
- setRoomState('disconnected');
727
- setAgentStateValue('idle');
728
- setCallFailure(null);
729
- setTurnErrorCode(null);
730
- setSynthesisError(null);
731
- setIsMuted(false);
732
- setMessagesState([]);
733
- setCurrentSpeaker(null);
734
- }, [teardown]);
735
-
736
- // ============== MUTE ==============
145
+ const call = callRef.current;
146
+ callRef.current = null;
147
+ call?.hangUp();
148
+ levels.reset();
149
+ if (mountedRef.current) setSnapshot(IDLE);
150
+ }, [levels]);
737
151
 
738
152
  const toggleMute = useCallback(() => {
739
- if (phaseRef.current === 'off') return;
740
- const next = !mutedRef.current;
741
- mutedRef.current = next;
742
- setIsMuted(next);
743
- if (next) {
744
- // What was half-said is dropped, not sent: muting mid-sentence means "not that".
745
- clearTimers();
746
- committingRef.current = false;
747
- heardRef.current = '';
748
- discardDraft();
749
- sessionRef.current?.abort();
750
- levels.setCapture(0);
751
- } else {
752
- startListeningRef.current();
753
- }
754
- }, [discardDraft, levels]);
755
-
756
- // ============== CLEANUP ON UNMOUNT ==============
153
+ const call = callRef.current;
154
+ if (call === null) return;
155
+ call.setMuted(!call.snapshot().isMuted);
156
+ }, []);
757
157
 
758
158
  useEffect(() => {
759
159
  mountedRef.current = true;
760
160
  return () => {
761
161
  mountedRef.current = false;
762
- teardown();
162
+ const call = callRef.current;
163
+ callRef.current = null;
164
+ call?.hangUp();
763
165
  };
764
- }, [teardown]);
166
+ }, []);
765
167
 
168
+ const { state } = snapshot;
169
+ const roomState = roomStateOf(state);
170
+ const errorCode: VoiceErrorCode | null = state.kind === 'error' ? state.code : null;
171
+ const endedReason: VoiceCloseReason | null = state.kind === 'ended' ? state.reason : null;
766
172
  const room: VoiceLevelSource = levels;
767
173
 
768
174
  return {
769
- /** The call's live levels, for `useAudioLevelMonitor`. Named for the room it replaced. */
175
+ /** The call's live levels, for `useAudioLevelMonitor`. */
770
176
  room,
177
+ /** The honest state machine (ADR 0017 §4). */
178
+ callState: state,
771
179
  roomState,
772
- agentState,
773
- isMuted,
774
- /** Why the call ended, in English. */
775
- error: callFailure?.message ?? null,
180
+ agentState: agentStateOf(state),
181
+ isMuted: snapshot.isMuted,
182
+ /** Why the call failed, in English. */
183
+ error: errorCode === null ? null : VOICE_ERROR_MESSAGES[errorCode],
776
184
  /** The same, as a code an app can translate. */
777
- errorCode: callFailure?.code ?? null,
778
- turnError: turnErrorCode === null ? null : VOICE_ERROR_MESSAGES[turnErrorCode],
779
- turnErrorCode,
780
- /**
781
- * When the call ended because a clip could not be made: the route's HTTP
782
- * status and typed code (`SpeechClipError`). `errorCode` says the same as a
783
- * `VoiceErrorCode` an app can translate.
784
- */
785
- synthesisError,
786
- messages,
787
- currentSpeaker,
185
+ errorCode,
186
+ /** Why the call ended without failing; `call-moved-elsewhere` is one. */
187
+ endedReason,
188
+ /** Milliseconds left once the server warned the call is about to end. */
189
+ endingSoonMs: snapshot.endingSoonMs,
190
+ /** The model serving the call (its name may be shown; the operator never is). */
191
+ model: snapshot.model,
192
+ /** The live transcript, both sides. The stored one is the conversation's. */
193
+ messages: snapshot.messages as VoiceMessage[],
194
+ currentSpeaker:
195
+ state.kind === 'speaking'
196
+ ? ('primary' as const)
197
+ : state.kind === 'user-speaking'
198
+ ? ('user' as const)
199
+ : null,
788
200
  connect,
789
201
  disconnect,
790
202
  toggleMute,