@alia.onl/sdk 13.2.0 → 15.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -0
- package/README.md +52 -20
- package/package.json +8 -4
- package/src/components/AliaAvatarIcon.tsx +30 -0
- package/src/components/AliaChatContent.tsx +4 -7
- package/src/components/AliaChatMessageList.tsx +2 -10
- package/src/components/AliaChatScreen.tsx +1 -1
- package/src/components/AliaChatSheet.tsx +15 -4
- package/src/components/AliaComposer.tsx +57 -11
- package/src/components/voice/VoiceControls.tsx +28 -4
- package/src/components/voice/VoiceSession.tsx +23 -7
- package/src/hooks/useAliaActivity.ts +31 -0
- package/src/hooks/useAliaChat.ts +14 -3
- package/src/hooks/useAmbientWave.ts +3 -3
- package/src/hooks/useAudioLevelMonitor.ts +2 -2
- package/src/hooks/useDictation.ts +295 -0
- package/src/hooks/useTTS.ts +24 -63
- package/src/hooks/useVoiceRoom.ts +148 -736
- package/src/index.ts +9 -3
- package/src/lib/activity-socket.ts +102 -0
- package/src/lib/activity-store.ts +53 -0
- package/src/lib/api-url.ts +2 -0
- package/src/lib/audio-level.ts +2 -2
- package/src/lib/dictation-recorder-types.ts +37 -0
- package/src/lib/dictation-recorder.native.ts +90 -0
- package/src/lib/dictation-recorder.ts +131 -0
- package/src/lib/dictation.ts +208 -0
- package/src/lib/labels.tsx +3 -0
- package/src/lib/pcm.ts +57 -0
- package/src/lib/speech-language.ts +1 -1
- package/src/lib/speech-messages.ts +43 -97
- package/src/lib/speech-synthesis.ts +5 -1
- package/src/lib/turn-state.ts +15 -0
- package/src/lib/voice-audio-engine-types.ts +59 -0
- package/src/lib/voice-audio-engine.native.ts +232 -0
- package/src/lib/voice-audio-engine.ts +350 -0
- package/src/lib/voice-call.ts +539 -0
- package/src/lib/voice-levels.ts +7 -3
- package/src/lib/voice-protocol.ts +309 -0
- package/src/types.ts +49 -4
- package/src/voice.ts +31 -12
- package/tsconfig.json +2 -0
- package/src/hooks/useSpeechToText.ts +0 -268
- package/src/lib/speech-recognition-types.ts +0 -108
- package/src/lib/speech-recognition.native.ts +0 -237
- package/src/lib/speech-recognition.ts +0 -308
- package/src/lib/voice-text.ts +0 -198
- package/src/lib/voice-turn.ts +0 -89
|
@@ -1,790 +1,202 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* A voice
|
|
2
|
+
* A voice call: speech to speech through Alia's `/v1/voice/session`
|
|
3
|
+
* (ADR 0017).
|
|
3
4
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
5
|
+
* Alia holds the realtime session with Kaana — instructions, memory, tools
|
|
6
|
+
* (run in Alia, never here), the transcript stored in the conversation, credits
|
|
7
|
+
* and typed errors. Turn detection and barge-in are the model's, on the server.
|
|
8
|
+
* This hook captures and plays PCM through the platform's audio engine
|
|
9
|
+
* (`lib/voice-audio-engine*`) and exposes a state it can prove (`callState`,
|
|
10
|
+
* see `VoiceCallState`), with `roomState` / `agentState` as the coarse
|
|
11
|
+
* projections the call bar, the ambient field and the sound effects read.
|
|
7
12
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* - **Thinking** sends the utterance through the ordinary chat path — a
|
|
11
|
-
* `VoiceTurnSender`, by default `POST /v1/chat/completions` marked
|
|
12
|
-
* `responseMode: 'voice'` — and streams the answer into the transcript.
|
|
13
|
-
* - **Speaking** synthesizes the answer through `POST /v1/audio/speech`
|
|
14
|
-
* sentence by sentence as it streams, so the first sentence plays while the
|
|
15
|
-
* rest is still being written, and the next clip is fetched while the
|
|
16
|
-
* current one plays.
|
|
17
|
-
* - **Barge-in:** the microphone stays open while Alia thinks and speaks. When
|
|
18
|
-
* the person says a few words that are not the answer echoing back, the turn
|
|
19
|
-
* is aborted — playback stops, the chat request is cancelled — and what they
|
|
20
|
-
* are saying becomes the next utterance.
|
|
21
|
-
* - **A voice that cannot speak ends the call.** When a clip cannot be made
|
|
22
|
-
* (`synthesisError`, typed from the route's code) or played, the turn stops
|
|
23
|
-
* speaking, its answer finishes streaming into the transcript, and the call
|
|
24
|
-
* ends with the reason as `errorCode` — never a call that listens on, mute.
|
|
25
|
-
*
|
|
26
|
-
* It replaced a LiveKit room (`POST /v1/voice/token`) whose realtime model and
|
|
27
|
-
* transcription called providers directly; Alia's only inference path is
|
|
28
|
-
* Alia → Oxy → Kaana, which serves chat and speech but no realtime session or
|
|
29
|
-
* transcription. The public shape — `connect` / `disconnect`, `roomState`,
|
|
30
|
-
* `agentState`, `messages`, `toggleMute` — is unchanged, so its consumers did
|
|
31
|
-
* not change with it.
|
|
13
|
+
* Nothing is recognized on the device: there is no browser or OS speech
|
|
14
|
+
* recognizer anywhere in a call.
|
|
32
15
|
*/
|
|
33
16
|
|
|
34
|
-
import {
|
|
17
|
+
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
|
35
18
|
import { useOxy } from '@oxy.so/services/ui/client';
|
|
36
|
-
import {
|
|
37
|
-
import
|
|
38
|
-
import {
|
|
39
|
-
isSpeechRecognitionAvailable,
|
|
40
|
-
chooseSpeechRecognizer,
|
|
41
|
-
requestSpeechRecognitionPermission,
|
|
42
|
-
startSpeechRecognition,
|
|
43
|
-
} from '../lib/speech-recognition';
|
|
44
|
-
import type {
|
|
45
|
-
SpeechRecognitionFailure,
|
|
46
|
-
SpeechRecognitionSession,
|
|
47
|
-
} from '../lib/speech-recognition-types';
|
|
48
|
-
import {
|
|
49
|
-
speechFailureCode,
|
|
50
|
-
synthesisFailureCode,
|
|
51
|
-
VOICE_ERROR_MESSAGES,
|
|
52
|
-
type VoiceErrorCode,
|
|
53
|
-
} from '../lib/speech-messages';
|
|
19
|
+
import type { AgentState, RoomState, VoiceCallState, VoiceMessage } from '../types';
|
|
20
|
+
import { VOICE_ERROR_MESSAGES, type VoiceErrorCode } from '../lib/speech-messages';
|
|
54
21
|
import { defaultSpeechLanguage } from '../lib/speech-language';
|
|
55
|
-
import {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
SpeechClipError,
|
|
59
|
-
type ProductVoice,
|
|
60
|
-
} from '../lib/speech-synthesis';
|
|
22
|
+
import { createVoiceAudioEngine } from '../lib/voice-audio-engine';
|
|
23
|
+
import type { VoiceAudioEngine } from '../lib/voice-audio-engine-types';
|
|
24
|
+
import { VoiceCall, type VoiceCallSnapshot, type VoiceSocketFactory } from '../lib/voice-call';
|
|
61
25
|
import { VoiceLevelChannel, type VoiceLevelSource } from '../lib/voice-levels';
|
|
62
|
-
import {
|
|
63
|
-
createAliaVoiceTurnSender,
|
|
64
|
-
type VoiceTurnMessage,
|
|
65
|
-
type VoiceTurnSender,
|
|
66
|
-
} from '../lib/voice-turn';
|
|
67
|
-
import {
|
|
68
|
-
dropEchoPrefix,
|
|
69
|
-
isInterruption,
|
|
70
|
-
stripTitleTags,
|
|
71
|
-
stripTitleTagsPartial,
|
|
72
|
-
takeSpeechChunks,
|
|
73
|
-
} from '../lib/voice-text';
|
|
26
|
+
import type { ProductVoice, VoiceCloseReason } from '../lib/voice-protocol';
|
|
74
27
|
|
|
75
28
|
const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
|
|
76
29
|
|
|
77
|
-
// ============== OPTIONS ==============
|
|
78
|
-
|
|
79
30
|
export interface UseVoiceRoomOptions {
|
|
80
31
|
apiUrl?: string;
|
|
81
|
-
/** Which of the product's two voices answers. Sent to `/v1/audio/speech` as is. */
|
|
82
|
-
voicePreference?: ProductVoice;
|
|
83
|
-
/** Bearer for speech synthesis; defaults to the surrounding Oxy session's. */
|
|
84
|
-
accessToken?: string;
|
|
85
|
-
/**
|
|
86
|
-
* Speech model for the answers. Omitted, the request carries no `model` and
|
|
87
|
-
* the server's own speech model answers.
|
|
88
|
-
*/
|
|
89
|
-
model?: string;
|
|
90
32
|
/**
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
* is given — that sender carries its own conversation's choice.
|
|
94
|
-
*/
|
|
95
|
-
chatModel?: string;
|
|
96
|
-
/**
|
|
97
|
-
* The agent this call belongs to, when it is a thread with one.
|
|
98
|
-
*
|
|
99
|
-
* Without it the call is ordinary Alia. Only the default sender reads it; a
|
|
100
|
-
* custom `sendTurn` is already bound to its conversation's agent.
|
|
33
|
+
* The conversation the call is stored in. Omitted, every call starts a
|
|
34
|
+
* conversation of its own (a fresh id per `connect`).
|
|
101
35
|
*/
|
|
36
|
+
conversationId?: string;
|
|
37
|
+
/** The agent this call talks to; omitted, Alia. */
|
|
102
38
|
agentId?: string;
|
|
103
|
-
/**
|
|
39
|
+
/** Which of the product's two voices answers. */
|
|
40
|
+
voicePreference?: ProductVoice;
|
|
41
|
+
/** Bearer for the call; defaults to the surrounding Oxy session's. */
|
|
42
|
+
accessToken?: string;
|
|
43
|
+
/** Language spoken (BCP-47). Defaults to the platform's. */
|
|
104
44
|
lang?: string;
|
|
105
|
-
/**
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
endOfUtteranceMs?: number;
|
|
113
|
-
/** Let the person interrupt by talking. On by default. */
|
|
114
|
-
bargeIn?: boolean;
|
|
45
|
+
/** Alia stored messages of the call: refresh the thread. */
|
|
46
|
+
onMessagesSaved?: (conversationId: string, messageIds: readonly string[]) => void;
|
|
47
|
+
/** A tool filed an action for the person's OK; its card is in the conversation. */
|
|
48
|
+
onApprovalRequested?: (toolName: string) => void;
|
|
49
|
+
/** Test seams: the device's audio and the socket. */
|
|
50
|
+
createAudioEngine?: () => VoiceAudioEngine;
|
|
51
|
+
openSocket?: VoiceSocketFactory;
|
|
115
52
|
}
|
|
116
53
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
/** A recognizer that keeps ending within this of starting is failing, not idling. */
|
|
125
|
-
const RAPID_END_MS = 400;
|
|
126
|
-
const MAX_RAPID_ENDS = 5;
|
|
54
|
+
const IDLE: VoiceCallSnapshot = {
|
|
55
|
+
state: { kind: 'idle' },
|
|
56
|
+
messages: [],
|
|
57
|
+
isMuted: false,
|
|
58
|
+
endingSoonMs: null,
|
|
59
|
+
model: null,
|
|
60
|
+
};
|
|
127
61
|
|
|
128
|
-
|
|
62
|
+
const openWebSocket: VoiceSocketFactory = (url) =>
|
|
63
|
+
new WebSocket(url) as unknown as ReturnType<VoiceSocketFactory>;
|
|
129
64
|
|
|
130
|
-
|
|
65
|
+
function newConversationId(): string {
|
|
66
|
+
const random = globalThis.crypto?.randomUUID?.();
|
|
67
|
+
return random ?? `voice-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
|
68
|
+
}
|
|
131
69
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
* Why this turn stopped speaking: a clip that could not be made (`synthesis`
|
|
148
|
-
* holds what the route said) or one the device could not play. The call ends
|
|
149
|
-
* with `code` once the answer has finished streaming.
|
|
150
|
-
*/
|
|
151
|
-
speechFailure: {
|
|
152
|
-
readonly code: VoiceErrorCode;
|
|
153
|
-
readonly synthesis: SpeechClipError | null;
|
|
154
|
-
} | null;
|
|
70
|
+
/** The coarse connection state the bar and the wave read. */
|
|
71
|
+
export function roomStateOf(state: VoiceCallState): RoomState {
|
|
72
|
+
switch (state.kind) {
|
|
73
|
+
case 'idle':
|
|
74
|
+
case 'ended':
|
|
75
|
+
return 'disconnected';
|
|
76
|
+
case 'connecting':
|
|
77
|
+
return 'connecting';
|
|
78
|
+
case 'reconnecting':
|
|
79
|
+
return 'reconnecting';
|
|
80
|
+
case 'error':
|
|
81
|
+
return 'error';
|
|
82
|
+
default:
|
|
83
|
+
return 'connected';
|
|
84
|
+
}
|
|
155
85
|
}
|
|
156
86
|
|
|
157
|
-
|
|
87
|
+
/** Who has the turn, coarsely. */
|
|
88
|
+
export function agentStateOf(state: VoiceCallState): AgentState {
|
|
89
|
+
switch (state.kind) {
|
|
90
|
+
case 'listening':
|
|
91
|
+
case 'user-speaking':
|
|
92
|
+
return 'listening';
|
|
93
|
+
case 'thinking':
|
|
94
|
+
case 'using-tool':
|
|
95
|
+
return 'thinking';
|
|
96
|
+
case 'speaking':
|
|
97
|
+
return 'speaking';
|
|
98
|
+
default:
|
|
99
|
+
return 'idle';
|
|
100
|
+
}
|
|
101
|
+
}
|
|
158
102
|
|
|
159
103
|
export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
160
|
-
const apiUrl = options.apiUrl || API_URL;
|
|
161
|
-
const voicePref: ProductVoice = options.voicePreference ?? 'female';
|
|
162
|
-
const speechModel = options.model;
|
|
163
|
-
const endOfUtteranceMs = options.endOfUtteranceMs ?? DEFAULT_END_OF_UTTERANCE_MS;
|
|
164
|
-
const bargeIn = options.bargeIn ?? true;
|
|
165
|
-
|
|
166
|
-
const [roomState, setRoomState] = useState<RoomState>('disconnected');
|
|
167
|
-
const [agentState, setAgentStateValue] = useState<AgentState>('idle');
|
|
168
|
-
const [isMuted, setIsMuted] = useState(false);
|
|
169
|
-
/**
|
|
170
|
-
* Why the call ended, as a code and as the sentence to show. The sentence is
|
|
171
|
-
* the code's English copy, except for `turn-failed`, which carries what the
|
|
172
|
-
* turn sender threw.
|
|
173
|
-
*/
|
|
174
|
-
const [callFailure, setCallFailure] = useState<{ code: VoiceErrorCode; message: string } | null>(
|
|
175
|
-
null,
|
|
176
|
-
);
|
|
177
|
-
/** A turn that failed without ending the call: no answer came back. */
|
|
178
|
-
const [turnErrorCode, setTurnErrorCode] = useState<VoiceErrorCode | null>(null);
|
|
179
|
-
/** The clip that could not be made, when that is what ended the call. */
|
|
180
|
-
const [synthesisError, setSynthesisError] = useState<SpeechClipError | null>(null);
|
|
181
|
-
const [messages, setMessagesState] = useState<VoiceMessage[]>([]);
|
|
182
|
-
const [currentSpeaker, setCurrentSpeaker] = useState<'primary' | 'user' | null>(null);
|
|
183
|
-
|
|
184
104
|
const { oxyServices } = useOxy();
|
|
185
|
-
|
|
186
105
|
const levels = useMemo(() => new VoiceLevelChannel(), []);
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
// arrive outside React, and a value closed over at `connect` would be stale.
|
|
190
|
-
const phaseRef = useRef<Phase>('off');
|
|
191
|
-
const mutedRef = useRef(false);
|
|
106
|
+
const [snapshot, setSnapshot] = useState<VoiceCallSnapshot>(IDLE);
|
|
107
|
+
const callRef = useRef<VoiceCall | null>(null);
|
|
192
108
|
const mountedRef = useRef(true);
|
|
193
|
-
const
|
|
194
|
-
|
|
195
|
-
/** The recognizer and tag the call listens with, chosen when it connects. */
|
|
196
|
-
const recognizerRef = useRef<{ lang: string; service?: string }>({ lang: '' });
|
|
197
|
-
const heardRef = useRef('');
|
|
198
|
-
const committingRef = useRef(false);
|
|
199
|
-
const silenceTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
|
200
|
-
const commitTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
|
201
|
-
const turnRef = useRef<ActiveTurn | null>(null);
|
|
202
|
-
const draftIdRef = useRef<string | null>(null);
|
|
203
|
-
/** What the call had said when the person cut in; this session's echo of it is not theirs. */
|
|
204
|
-
const echoContextRef = useRef('');
|
|
205
|
-
const sessionStartedAtRef = useRef(0);
|
|
206
|
-
const rapidEndsRef = useRef(0);
|
|
207
|
-
const sessionFailedRef = useRef<SpeechRecognitionFailure | null>(null);
|
|
208
|
-
const msgIdRef = useRef(0);
|
|
209
|
-
const sessionPrefixRef = useRef(`vm-${Date.now().toString(36)}`);
|
|
210
|
-
|
|
211
|
-
const config = {
|
|
212
|
-
apiUrl,
|
|
213
|
-
voicePref,
|
|
214
|
-
speechModel,
|
|
215
|
-
endOfUtteranceMs,
|
|
216
|
-
bargeIn,
|
|
217
|
-
lang: options.lang ?? defaultSpeechLanguage(),
|
|
218
|
-
accessToken: options.accessToken,
|
|
219
|
-
sendTurn:
|
|
220
|
-
options.sendTurn ??
|
|
221
|
-
createAliaVoiceTurnSender({
|
|
222
|
-
oxyServices,
|
|
223
|
-
apiUrl,
|
|
224
|
-
...(options.chatModel === undefined ? {} : { model: options.chatModel }),
|
|
225
|
-
...(options.agentId === undefined ? {} : { agentId: options.agentId }),
|
|
226
|
-
}),
|
|
227
|
-
};
|
|
228
|
-
const configRef = useRef(config);
|
|
229
|
-
configRef.current = config;
|
|
230
|
-
|
|
231
|
-
// ============== STATE HELPERS ==============
|
|
232
|
-
|
|
233
|
-
const setAgent = useCallback((phase: Phase) => {
|
|
234
|
-
phaseRef.current = phase;
|
|
235
|
-
if (!mountedRef.current) return;
|
|
236
|
-
setAgentStateValue(phase === 'off' ? 'idle' : phase);
|
|
237
|
-
setCurrentSpeaker(phase === 'thinking' || phase === 'speaking' ? 'primary' : null);
|
|
238
|
-
}, []);
|
|
239
|
-
|
|
240
|
-
const updateMessages = useCallback((update: (previous: VoiceMessage[]) => VoiceMessage[]) => {
|
|
241
|
-
messagesRef.current = update(messagesRef.current);
|
|
242
|
-
if (mountedRef.current) setMessagesState(messagesRef.current);
|
|
243
|
-
}, []);
|
|
244
|
-
|
|
245
|
-
const nextId = (): string => {
|
|
246
|
-
msgIdRef.current += 1;
|
|
247
|
-
return `${sessionPrefixRef.current}-${msgIdRef.current}`;
|
|
248
|
-
};
|
|
249
|
-
|
|
250
|
-
const patchMessage = useCallback(
|
|
251
|
-
(id: string, patch: Partial<VoiceMessage>) => {
|
|
252
|
-
updateMessages((previous) =>
|
|
253
|
-
previous.map((message) => (message.id === id ? { ...message, ...patch } : message)),
|
|
254
|
-
);
|
|
255
|
-
},
|
|
256
|
-
[updateMessages],
|
|
257
|
-
);
|
|
109
|
+
const optionsRef = useRef(options);
|
|
110
|
+
optionsRef.current = options;
|
|
258
111
|
|
|
259
112
|
const getToken = useCallback((): string | null => {
|
|
260
|
-
const { accessToken } =
|
|
113
|
+
const { accessToken } = optionsRef.current;
|
|
261
114
|
if (accessToken) return accessToken;
|
|
262
|
-
return oxyServices.session.accessToken;
|
|
115
|
+
return oxyServices.session.accessToken ?? null;
|
|
263
116
|
}, [oxyServices]);
|
|
264
117
|
|
|
265
|
-
const clearTimers = (): void => {
|
|
266
|
-
if (silenceTimerRef.current !== null) clearTimeout(silenceTimerRef.current);
|
|
267
|
-
if (commitTimerRef.current !== null) clearTimeout(commitTimerRef.current);
|
|
268
|
-
silenceTimerRef.current = null;
|
|
269
|
-
commitTimerRef.current = null;
|
|
270
|
-
};
|
|
271
|
-
|
|
272
|
-
// ============== TEARDOWN ==============
|
|
273
|
-
|
|
274
|
-
const endTurn = useCallback(
|
|
275
|
-
(turn: ActiveTurn) => {
|
|
276
|
-
turn.controller.abort();
|
|
277
|
-
turn.closed = true;
|
|
278
|
-
turn.wake?.();
|
|
279
|
-
if (turnRef.current === turn) turnRef.current = null;
|
|
280
|
-
const message = messagesRef.current.find((candidate) => candidate.id === turn.assistantId);
|
|
281
|
-
if (message?.isStreaming) {
|
|
282
|
-
if (message.content.trim() === '') {
|
|
283
|
-
updateMessages((previous) =>
|
|
284
|
-
previous.filter((candidate) => candidate.id !== turn.assistantId),
|
|
285
|
-
);
|
|
286
|
-
} else {
|
|
287
|
-
patchMessage(turn.assistantId, { isStreaming: false });
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
levels.setPlayback(0);
|
|
291
|
-
},
|
|
292
|
-
[levels, patchMessage, updateMessages],
|
|
293
|
-
);
|
|
294
|
-
|
|
295
|
-
const discardDraft = useCallback(() => {
|
|
296
|
-
const draftId = draftIdRef.current;
|
|
297
|
-
draftIdRef.current = null;
|
|
298
|
-
if (draftId !== null)
|
|
299
|
-
updateMessages((previous) => previous.filter((message) => message.id !== draftId));
|
|
300
|
-
}, [updateMessages]);
|
|
301
|
-
|
|
302
|
-
/** Stop everything; the caller decides what state the room is left in. */
|
|
303
|
-
const teardown = useCallback(() => {
|
|
304
|
-
phaseRef.current = 'off';
|
|
305
|
-
clearTimers();
|
|
306
|
-
committingRef.current = false;
|
|
307
|
-
const turn = turnRef.current;
|
|
308
|
-
if (turn !== null) endTurn(turn);
|
|
309
|
-
sessionRef.current?.abort();
|
|
310
|
-
sessionRef.current = null;
|
|
311
|
-
heardRef.current = '';
|
|
312
|
-
levels.reset();
|
|
313
|
-
}, [endTurn, levels]);
|
|
314
|
-
|
|
315
|
-
const fail = useCallback(
|
|
316
|
-
(code: VoiceErrorCode, message: string = VOICE_ERROR_MESSAGES[code]) => {
|
|
317
|
-
teardown();
|
|
318
|
-
discardDraft();
|
|
319
|
-
if (!mountedRef.current) return;
|
|
320
|
-
setCallFailure({ code, message });
|
|
321
|
-
setRoomState('error');
|
|
322
|
-
setAgentStateValue('idle');
|
|
323
|
-
setCurrentSpeaker(null);
|
|
324
|
-
},
|
|
325
|
-
[discardDraft, teardown],
|
|
326
|
-
);
|
|
327
|
-
|
|
328
|
-
// ============== SPEAKING ==============
|
|
329
|
-
|
|
330
|
-
/** The first reason a turn could not speak is the one the call ends with. */
|
|
331
|
-
const stopSpeaking = useCallback(
|
|
332
|
-
(turn: ActiveTurn, code: VoiceErrorCode, synthesis: SpeechClipError | null) => {
|
|
333
|
-
turn.speechFailure ??= { code, synthesis };
|
|
334
|
-
},
|
|
335
|
-
[],
|
|
336
|
-
);
|
|
337
|
-
|
|
338
|
-
const synthesize = useCallback(
|
|
339
|
-
(turn: ActiveTurn, text: string): Promise<string | null> => {
|
|
340
|
-
const token = getToken();
|
|
341
|
-
if (token === null) {
|
|
342
|
-
const failure = new SpeechClipError(VOICE_ERROR_MESSAGES['not-authenticated'], {
|
|
343
|
-
status: null,
|
|
344
|
-
code: 'NOT_AUTHENTICATED',
|
|
345
|
-
});
|
|
346
|
-
stopSpeaking(turn, synthesisFailureCode(failure), failure);
|
|
347
|
-
return Promise.resolve(null);
|
|
348
|
-
}
|
|
349
|
-
const { apiUrl: url, speechModel: model, voicePref: voice } = configRef.current;
|
|
350
|
-
return requestSpeechClip({
|
|
351
|
-
apiUrl: url,
|
|
352
|
-
token,
|
|
353
|
-
model,
|
|
354
|
-
input: text,
|
|
355
|
-
voice,
|
|
356
|
-
signal: turn.controller.signal,
|
|
357
|
-
}).catch((caught: unknown) => {
|
|
358
|
-
if (turn.controller.signal.aborted) return null;
|
|
359
|
-
const failure =
|
|
360
|
-
caught instanceof SpeechClipError
|
|
361
|
-
? caught
|
|
362
|
-
: new SpeechClipError(errorMessage(caught, 'TTS failed'), { status: null, code: null });
|
|
363
|
-
console.error('[useVoiceRoom] Speech synthesis failed:', {
|
|
364
|
-
status: failure.status,
|
|
365
|
-
code: failure.code,
|
|
366
|
-
requestId: failure.requestId,
|
|
367
|
-
});
|
|
368
|
-
stopSpeaking(turn, synthesisFailureCode(failure), failure);
|
|
369
|
-
return null;
|
|
370
|
-
});
|
|
371
|
-
},
|
|
372
|
-
[getToken, stopSpeaking],
|
|
373
|
-
);
|
|
374
|
-
|
|
375
|
-
/**
|
|
376
|
-
* Play the turn's clips in order as they arrive, fetching one ahead. Returns
|
|
377
|
-
* early, without skipping to a later sentence, once a clip cannot be made or
|
|
378
|
-
* played.
|
|
379
|
-
*/
|
|
380
|
-
const playTurn = useCallback(
|
|
381
|
-
async (turn: ActiveTurn): Promise<void> => {
|
|
382
|
-
const signal = turn.controller.signal;
|
|
383
|
-
for (let index = 0; ; index += 1) {
|
|
384
|
-
while (index >= turn.clips.length && !turn.closed) {
|
|
385
|
-
await new Promise<void>((resolve) => {
|
|
386
|
-
turn.wake = resolve;
|
|
387
|
-
});
|
|
388
|
-
turn.wake = null;
|
|
389
|
-
}
|
|
390
|
-
if (signal.aborted || index >= turn.clips.length) return;
|
|
391
|
-
const item = turn.clips[index];
|
|
392
|
-
item.clip ??= synthesize(turn, item.text);
|
|
393
|
-
const next = turn.clips[index + 1];
|
|
394
|
-
if (next !== undefined) next.clip ??= synthesize(turn, next.text);
|
|
395
|
-
|
|
396
|
-
const uri = await item.clip;
|
|
397
|
-
if (signal.aborted) return;
|
|
398
|
-
if (uri === null) {
|
|
399
|
-
setAgent('thinking');
|
|
400
|
-
levels.setPlayback(0);
|
|
401
|
-
return;
|
|
402
|
-
}
|
|
403
|
-
turn.speaking = `${turn.speaking} ${item.text}`;
|
|
404
|
-
setAgent('speaking');
|
|
405
|
-
try {
|
|
406
|
-
await playSpeechClip(uri, { signal, onLevel: (level) => levels.setPlayback(level) });
|
|
407
|
-
} catch (caught: unknown) {
|
|
408
|
-
if (signal.aborted) return;
|
|
409
|
-
console.error('[useVoiceRoom] Playback failed:', caught);
|
|
410
|
-
stopSpeaking(turn, 'not-played', null);
|
|
411
|
-
setAgent('thinking');
|
|
412
|
-
return;
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
},
|
|
416
|
-
[levels, setAgent, stopSpeaking, synthesize],
|
|
417
|
-
);
|
|
418
|
-
|
|
419
|
-
const queueChunks = useCallback((turn: ActiveTurn, final: boolean) => {
|
|
420
|
-
const pending = turn.text.slice(turn.consumed);
|
|
421
|
-
const { chunks, rest } = takeSpeechChunks(pending, { final, isFirst: turn.chunkCount === 0 });
|
|
422
|
-
turn.consumed = turn.text.length - rest.length;
|
|
423
|
-
for (const text of chunks) {
|
|
424
|
-
turn.chunkCount += 1;
|
|
425
|
-
// The next clip is fetched now if the player is waiting on it; the
|
|
426
|
-
// player fetches the rest one ahead of what it is playing.
|
|
427
|
-
turn.clips.push({ text, clip: null });
|
|
428
|
-
}
|
|
429
|
-
if (chunks.length > 0) turn.wake?.();
|
|
430
|
-
}, []);
|
|
431
|
-
|
|
432
|
-
// ============== LISTENING ==============
|
|
433
|
-
|
|
434
|
-
// Declared through refs so the recognizer callbacks and the turn runner can
|
|
435
|
-
// call each other without a cycle in their dependency lists.
|
|
436
|
-
const startListeningRef = useRef<() => void>(() => undefined);
|
|
437
|
-
const runTurnRef = useRef<(text: string) => void>(() => undefined);
|
|
438
|
-
|
|
439
|
-
const commitUtterance = useCallback(() => {
|
|
440
|
-
const session = sessionRef.current;
|
|
441
|
-
if (session === null || committingRef.current) return;
|
|
442
|
-
committingRef.current = true;
|
|
443
|
-
session.stop();
|
|
444
|
-
commitTimerRef.current = setTimeout(() => {
|
|
445
|
-
commitTimerRef.current = null;
|
|
446
|
-
sessionRef.current?.abort();
|
|
447
|
-
}, COMMIT_TIMEOUT_MS);
|
|
448
|
-
}, []);
|
|
449
|
-
|
|
450
|
-
const showDraft = useCallback(
|
|
451
|
-
(transcript: string) => {
|
|
452
|
-
const draftId = draftIdRef.current;
|
|
453
|
-
if (draftId !== null) {
|
|
454
|
-
patchMessage(draftId, { content: transcript });
|
|
455
|
-
return;
|
|
456
|
-
}
|
|
457
|
-
const id = nextId();
|
|
458
|
-
draftIdRef.current = id;
|
|
459
|
-
updateMessages((previous) => [
|
|
460
|
-
...previous,
|
|
461
|
-
{ id, role: 'user', content: transcript, timestamp: Date.now(), isStreaming: true },
|
|
462
|
-
]);
|
|
463
|
-
if (mountedRef.current) setCurrentSpeaker('user');
|
|
464
|
-
},
|
|
465
|
-
[patchMessage, updateMessages],
|
|
466
|
-
);
|
|
467
|
-
|
|
468
|
-
const startListening = useCallback(() => {
|
|
469
|
-
if (phaseRef.current === 'off' || mutedRef.current || sessionRef.current !== null) return;
|
|
470
|
-
heardRef.current = '';
|
|
471
|
-
committingRef.current = false;
|
|
472
|
-
sessionFailedRef.current = null;
|
|
473
|
-
echoContextRef.current = '';
|
|
474
|
-
sessionStartedAtRef.current = Date.now();
|
|
475
|
-
|
|
476
|
-
sessionRef.current = startSpeechRecognition(
|
|
477
|
-
{ ...recognizerRef.current, echoCancellation: true },
|
|
478
|
-
{
|
|
479
|
-
onResult: ({ transcript }) => {
|
|
480
|
-
if (phaseRef.current === 'off' || committingRef.current) return;
|
|
481
|
-
|
|
482
|
-
const turn = turnRef.current;
|
|
483
|
-
if (turn !== null) {
|
|
484
|
-
// Alia is thinking or speaking: this is the person cutting in, or
|
|
485
|
-
// the answer coming back through the microphone.
|
|
486
|
-
heardRef.current = '';
|
|
487
|
-
if (!configRef.current.bargeIn) return;
|
|
488
|
-
if (!isInterruption(transcript, turn.speaking, BARGE_IN_MIN_WORDS)) return;
|
|
489
|
-
echoContextRef.current = turn.speaking;
|
|
490
|
-
endTurn(turn);
|
|
491
|
-
setAgent('listening');
|
|
492
|
-
}
|
|
493
|
-
|
|
494
|
-
const utterance = dropEchoPrefix(transcript, echoContextRef.current);
|
|
495
|
-
heardRef.current = utterance;
|
|
496
|
-
if (utterance === '') return;
|
|
497
|
-
showDraft(utterance);
|
|
498
|
-
if (silenceTimerRef.current !== null) clearTimeout(silenceTimerRef.current);
|
|
499
|
-
silenceTimerRef.current = setTimeout(() => {
|
|
500
|
-
silenceTimerRef.current = null;
|
|
501
|
-
commitUtterance();
|
|
502
|
-
}, configRef.current.endOfUtteranceMs);
|
|
503
|
-
},
|
|
504
|
-
onLevel: (level) => {
|
|
505
|
-
levels.setCapture(mutedRef.current ? 0 : level);
|
|
506
|
-
},
|
|
507
|
-
onError: (failure) => {
|
|
508
|
-
if (failure.code === 'no-speech' || failure.code === 'aborted') return;
|
|
509
|
-
sessionFailedRef.current = failure;
|
|
510
|
-
},
|
|
511
|
-
onEnd: () => {
|
|
512
|
-
sessionRef.current = null;
|
|
513
|
-
clearTimers();
|
|
514
|
-
const committing = committingRef.current;
|
|
515
|
-
committingRef.current = false;
|
|
516
|
-
const heard = heardRef.current.trim();
|
|
517
|
-
heardRef.current = '';
|
|
518
|
-
if (phaseRef.current === 'off') return;
|
|
519
|
-
|
|
520
|
-
const failure = sessionFailedRef.current;
|
|
521
|
-
if (failure !== null && failure.code !== 'other') {
|
|
522
|
-
fail(speechFailureCode(failure) ?? 'speech-failed');
|
|
523
|
-
return;
|
|
524
|
-
}
|
|
525
|
-
|
|
526
|
-
// An utterance in progress when the engine ended is still an utterance.
|
|
527
|
-
if (
|
|
528
|
-
(committing || draftIdRef.current !== null) &&
|
|
529
|
-
heard !== '' &&
|
|
530
|
-
turnRef.current === null
|
|
531
|
-
) {
|
|
532
|
-
rapidEndsRef.current = 0;
|
|
533
|
-
runTurnRef.current(heard);
|
|
534
|
-
return;
|
|
535
|
-
}
|
|
536
|
-
|
|
537
|
-
if (Date.now() - sessionStartedAtRef.current < RAPID_END_MS) {
|
|
538
|
-
rapidEndsRef.current += 1;
|
|
539
|
-
if (rapidEndsRef.current >= MAX_RAPID_ENDS) {
|
|
540
|
-
fail(speechFailureCode(failure ?? { code: 'other' }) ?? 'speech-failed');
|
|
541
|
-
return;
|
|
542
|
-
}
|
|
543
|
-
} else {
|
|
544
|
-
rapidEndsRef.current = 0;
|
|
545
|
-
}
|
|
546
|
-
startListeningRef.current();
|
|
547
|
-
},
|
|
548
|
-
},
|
|
549
|
-
);
|
|
550
|
-
}, [commitUtterance, endTurn, fail, levels, setAgent, showDraft]);
|
|
551
|
-
startListeningRef.current = startListening;
|
|
552
|
-
|
|
553
|
-
// ============== THINKING ==============
|
|
554
|
-
|
|
555
|
-
const runTurn = useCallback(
|
|
556
|
-
(text: string) => {
|
|
557
|
-
// Settle the person's words as a message of their own.
|
|
558
|
-
const draftId = draftIdRef.current;
|
|
559
|
-
draftIdRef.current = null;
|
|
560
|
-
if (draftId !== null) {
|
|
561
|
-
patchMessage(draftId, { content: text, isStreaming: false });
|
|
562
|
-
} else {
|
|
563
|
-
updateMessages((previous) => [
|
|
564
|
-
...previous,
|
|
565
|
-
{ id: nextId(), role: 'user', content: text, timestamp: Date.now(), isStreaming: false },
|
|
566
|
-
]);
|
|
567
|
-
}
|
|
568
|
-
|
|
569
|
-
const history: VoiceTurnMessage[] = messagesRef.current
|
|
570
|
-
.filter((message) => message.content.trim() !== '' && message.id !== draftId)
|
|
571
|
-
.map((message) => ({ role: message.role, content: message.content }))
|
|
572
|
-
// The turn itself is `text`, not history.
|
|
573
|
-
.slice(0, draftId === null ? -1 : undefined);
|
|
574
|
-
|
|
575
|
-
const assistantId = nextId();
|
|
576
|
-
updateMessages((previous) => [
|
|
577
|
-
...previous,
|
|
578
|
-
{
|
|
579
|
-
id: assistantId,
|
|
580
|
-
role: 'assistant',
|
|
581
|
-
speaker: 'primary',
|
|
582
|
-
content: '',
|
|
583
|
-
timestamp: Date.now(),
|
|
584
|
-
isStreaming: true,
|
|
585
|
-
},
|
|
586
|
-
]);
|
|
587
|
-
|
|
588
|
-
const turn: ActiveTurn = {
|
|
589
|
-
controller: new AbortController(),
|
|
590
|
-
assistantId,
|
|
591
|
-
consumed: 0,
|
|
592
|
-
chunkCount: 0,
|
|
593
|
-
text: '',
|
|
594
|
-
speaking: '',
|
|
595
|
-
clips: [],
|
|
596
|
-
closed: false,
|
|
597
|
-
wake: null,
|
|
598
|
-
speechFailure: null,
|
|
599
|
-
};
|
|
600
|
-
turnRef.current = turn;
|
|
601
|
-
setAgent('thinking');
|
|
602
|
-
if (mountedRef.current) setTurnErrorCode(null);
|
|
603
|
-
|
|
604
|
-
// The microphone stays open through thinking and speaking, for barge-in.
|
|
605
|
-
startListeningRef.current();
|
|
606
|
-
|
|
607
|
-
const playing = playTurn(turn);
|
|
608
|
-
const signal = turn.controller.signal;
|
|
609
|
-
|
|
610
|
-
void (async () => {
|
|
611
|
-
let lastText = '';
|
|
612
|
-
try {
|
|
613
|
-
await configRef.current.sendTurn({
|
|
614
|
-
text,
|
|
615
|
-
history,
|
|
616
|
-
signal,
|
|
617
|
-
onText: (answerSoFar) => {
|
|
618
|
-
if (signal.aborted) return;
|
|
619
|
-
lastText = answerSoFar;
|
|
620
|
-
turn.text = stripTitleTagsPartial(answerSoFar, { trim: false });
|
|
621
|
-
patchMessage(assistantId, { content: turn.text.trim() });
|
|
622
|
-
queueChunks(turn, false);
|
|
623
|
-
},
|
|
624
|
-
});
|
|
625
|
-
} catch (caught: unknown) {
|
|
626
|
-
if (signal.aborted) return;
|
|
627
|
-
console.error('[useVoiceRoom] Turn failed:', caught);
|
|
628
|
-
fail('turn-failed', errorMessage(caught, VOICE_ERROR_MESSAGES['turn-failed']));
|
|
629
|
-
return;
|
|
630
|
-
}
|
|
631
|
-
if (signal.aborted) return;
|
|
632
|
-
|
|
633
|
-
turn.text = stripTitleTags(lastText, { trim: false });
|
|
634
|
-
patchMessage(assistantId, { content: turn.text.trim(), isStreaming: false });
|
|
635
|
-
queueChunks(turn, true);
|
|
636
|
-
turn.closed = true;
|
|
637
|
-
turn.wake?.();
|
|
638
|
-
await playing;
|
|
639
|
-
if (signal.aborted || turnRef.current !== turn) return;
|
|
640
|
-
|
|
641
|
-
// The answer is in the transcript; a call that cannot say it ends here,
|
|
642
|
-
// rather than listening on in silence.
|
|
643
|
-
const speechFailure = turn.speechFailure;
|
|
644
|
-
if (speechFailure !== null) {
|
|
645
|
-
if (mountedRef.current) setSynthesisError(speechFailure.synthesis);
|
|
646
|
-
fail(speechFailure.code);
|
|
647
|
-
return;
|
|
648
|
-
}
|
|
649
|
-
|
|
650
|
-
turnRef.current = null;
|
|
651
|
-
levels.setPlayback(0);
|
|
652
|
-
// The session that listened through the answer has heard the answer; a
|
|
653
|
-
// fresh one starts the next utterance clean.
|
|
654
|
-
sessionRef.current?.abort();
|
|
655
|
-
if (turn.text.trim() === '') {
|
|
656
|
-
updateMessages((previous) => previous.filter((message) => message.id !== assistantId));
|
|
657
|
-
if (mountedRef.current) setTurnErrorCode('no-answer');
|
|
658
|
-
}
|
|
659
|
-
setAgent('listening');
|
|
660
|
-
startListeningRef.current();
|
|
661
|
-
})();
|
|
662
|
-
},
|
|
663
|
-
[fail, levels, patchMessage, playTurn, queueChunks, setAgent, updateMessages],
|
|
664
|
-
);
|
|
665
|
-
runTurnRef.current = runTurn;
|
|
666
|
-
|
|
667
|
-
// ============== CONNECT ==============
|
|
668
|
-
|
|
669
118
|
const connect = useCallback(async () => {
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
if (refusal !== null) {
|
|
696
|
-
const code = speechFailureCode(refusal) ?? 'microphone-denied';
|
|
697
|
-
setCallFailure({ code, message: VOICE_ERROR_MESSAGES[code] });
|
|
698
|
-
setRoomState('error');
|
|
699
|
-
return;
|
|
700
|
-
}
|
|
701
|
-
|
|
702
|
-
// The recognizer that knows the call's language, as dictation picks it.
|
|
703
|
-
const choice = await chooseSpeechRecognizer(configRef.current.lang);
|
|
704
|
-
if (!mountedRef.current) return;
|
|
705
|
-
if ('failure' in choice) {
|
|
706
|
-
const code = speechFailureCode(choice.failure) ?? 'speech-language';
|
|
707
|
-
setCallFailure({ code, message: VOICE_ERROR_MESSAGES[code] });
|
|
708
|
-
setRoomState('error');
|
|
709
|
-
return;
|
|
710
|
-
}
|
|
711
|
-
recognizerRef.current = { lang: choice.lang, service: choice.service };
|
|
712
|
-
|
|
713
|
-
rapidEndsRef.current = 0;
|
|
714
|
-
setRoomState('connected');
|
|
715
|
-
setAgent('listening');
|
|
716
|
-
startListeningRef.current();
|
|
717
|
-
}, [getToken, setAgent]);
|
|
718
|
-
|
|
719
|
-
// ============== DISCONNECT ==============
|
|
720
|
-
|
|
119
|
+
const live = callRef.current?.snapshot().state.kind;
|
|
120
|
+
if (live !== undefined && live !== 'ended' && live !== 'error') return;
|
|
121
|
+
const current = optionsRef.current;
|
|
122
|
+
const call = new VoiceCall({
|
|
123
|
+
apiUrl: current.apiUrl || API_URL,
|
|
124
|
+
getToken,
|
|
125
|
+
conversationId: current.conversationId ?? newConversationId(),
|
|
126
|
+
...(current.agentId === undefined ? {} : { agentId: current.agentId }),
|
|
127
|
+
voice: current.voicePreference ?? 'female',
|
|
128
|
+
locale: current.lang ?? defaultSpeechLanguage(),
|
|
129
|
+
createEngine: current.createAudioEngine ?? createVoiceAudioEngine,
|
|
130
|
+
openSocket: current.openSocket ?? openWebSocket,
|
|
131
|
+
levels,
|
|
132
|
+
onChange: (next) => {
|
|
133
|
+
if (mountedRef.current && callRef.current === call) setSnapshot(next);
|
|
134
|
+
},
|
|
135
|
+
onMessagesSaved: (conversationId, ids) =>
|
|
136
|
+
optionsRef.current.onMessagesSaved?.(conversationId, ids),
|
|
137
|
+
onApprovalRequested: (toolName) => optionsRef.current.onApprovalRequested?.(toolName),
|
|
138
|
+
});
|
|
139
|
+
callRef.current = call;
|
|
140
|
+
await call.start();
|
|
141
|
+
}, [getToken, levels]);
|
|
142
|
+
|
|
143
|
+
/** Hang up and forget the call: the room is back to `disconnected`. */
|
|
721
144
|
const disconnect = useCallback(() => {
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
setCallFailure(null);
|
|
729
|
-
setTurnErrorCode(null);
|
|
730
|
-
setSynthesisError(null);
|
|
731
|
-
setIsMuted(false);
|
|
732
|
-
setMessagesState([]);
|
|
733
|
-
setCurrentSpeaker(null);
|
|
734
|
-
}, [teardown]);
|
|
735
|
-
|
|
736
|
-
// ============== MUTE ==============
|
|
145
|
+
const call = callRef.current;
|
|
146
|
+
callRef.current = null;
|
|
147
|
+
call?.hangUp();
|
|
148
|
+
levels.reset();
|
|
149
|
+
if (mountedRef.current) setSnapshot(IDLE);
|
|
150
|
+
}, [levels]);
|
|
737
151
|
|
|
738
152
|
const toggleMute = useCallback(() => {
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
if (next) {
|
|
744
|
-
// What was half-said is dropped, not sent: muting mid-sentence means "not that".
|
|
745
|
-
clearTimers();
|
|
746
|
-
committingRef.current = false;
|
|
747
|
-
heardRef.current = '';
|
|
748
|
-
discardDraft();
|
|
749
|
-
sessionRef.current?.abort();
|
|
750
|
-
levels.setCapture(0);
|
|
751
|
-
} else {
|
|
752
|
-
startListeningRef.current();
|
|
753
|
-
}
|
|
754
|
-
}, [discardDraft, levels]);
|
|
755
|
-
|
|
756
|
-
// ============== CLEANUP ON UNMOUNT ==============
|
|
153
|
+
const call = callRef.current;
|
|
154
|
+
if (call === null) return;
|
|
155
|
+
call.setMuted(!call.snapshot().isMuted);
|
|
156
|
+
}, []);
|
|
757
157
|
|
|
758
158
|
useEffect(() => {
|
|
759
159
|
mountedRef.current = true;
|
|
760
160
|
return () => {
|
|
761
161
|
mountedRef.current = false;
|
|
762
|
-
|
|
162
|
+
const call = callRef.current;
|
|
163
|
+
callRef.current = null;
|
|
164
|
+
call?.hangUp();
|
|
763
165
|
};
|
|
764
|
-
}, [
|
|
166
|
+
}, []);
|
|
765
167
|
|
|
168
|
+
const { state } = snapshot;
|
|
169
|
+
const roomState = roomStateOf(state);
|
|
170
|
+
const errorCode: VoiceErrorCode | null = state.kind === 'error' ? state.code : null;
|
|
171
|
+
const endedReason: VoiceCloseReason | null = state.kind === 'ended' ? state.reason : null;
|
|
766
172
|
const room: VoiceLevelSource = levels;
|
|
767
173
|
|
|
768
174
|
return {
|
|
769
|
-
/** The call's live levels, for `useAudioLevelMonitor`.
|
|
175
|
+
/** The call's live levels, for `useAudioLevelMonitor`. */
|
|
770
176
|
room,
|
|
177
|
+
/** The honest state machine (ADR 0017 §4). */
|
|
178
|
+
callState: state,
|
|
771
179
|
roomState,
|
|
772
|
-
agentState,
|
|
773
|
-
isMuted,
|
|
774
|
-
/** Why the call
|
|
775
|
-
error:
|
|
180
|
+
agentState: agentStateOf(state),
|
|
181
|
+
isMuted: snapshot.isMuted,
|
|
182
|
+
/** Why the call failed, in English. */
|
|
183
|
+
error: errorCode === null ? null : VOICE_ERROR_MESSAGES[errorCode],
|
|
776
184
|
/** The same, as a code an app can translate. */
|
|
777
|
-
errorCode
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
/**
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
185
|
+
errorCode,
|
|
186
|
+
/** Why the call ended without failing; `call-moved-elsewhere` is one. */
|
|
187
|
+
endedReason,
|
|
188
|
+
/** Milliseconds left once the server warned the call is about to end. */
|
|
189
|
+
endingSoonMs: snapshot.endingSoonMs,
|
|
190
|
+
/** The model serving the call (its name may be shown; the operator never is). */
|
|
191
|
+
model: snapshot.model,
|
|
192
|
+
/** The live transcript, both sides. The stored one is the conversation's. */
|
|
193
|
+
messages: snapshot.messages as VoiceMessage[],
|
|
194
|
+
currentSpeaker:
|
|
195
|
+
state.kind === 'speaking'
|
|
196
|
+
? ('primary' as const)
|
|
197
|
+
: state.kind === 'user-speaking'
|
|
198
|
+
? ('user' as const)
|
|
199
|
+
: null,
|
|
788
200
|
connect,
|
|
789
201
|
disconnect,
|
|
790
202
|
toggleMute,
|