@alia.onl/sdk 7.2.7 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/README.md +234 -0
- package/package.json +9 -13
- package/src/components/AliaChatContent.tsx +0 -1
- package/src/components/AliaChatMessageList.tsx +0 -4
- package/src/components/ResearchProgressCard.tsx +3 -0
- package/src/components/ui/prompt-input/mic-button.tsx +2 -3
- package/src/components/ui/prompt-input/prompt-input.tsx +1 -4
- package/src/components/voice/VoiceControls.tsx +3 -54
- package/src/components/voice/VoiceSession.tsx +6 -19
- package/src/hooks/useAliaChat.ts +16 -1
- package/src/hooks/useAudioLevelMonitor.ts +26 -122
- package/src/hooks/useSpeechToText.ts +165 -150
- package/src/hooks/useTTS.ts +29 -29
- package/src/hooks/useVoiceRoom.ts +553 -409
- package/src/index.ts +0 -14
- package/src/lib/audio-level.ts +3 -2
- package/src/lib/chat-transport.ts +11 -0
- package/src/lib/speech-language.ts +20 -0
- package/src/lib/speech-messages.ts +31 -0
- package/src/lib/speech-recognition-types.ts +75 -0
- package/src/lib/speech-recognition.native.ts +171 -0
- package/src/lib/speech-recognition.ts +279 -0
- package/src/lib/speech-synthesis.ts +122 -0
- package/src/lib/voice-levels.ts +60 -0
- package/src/lib/voice-text.ts +192 -0
- package/src/lib/voice-turn.ts +87 -0
- package/src/types.ts +3 -3
- package/src/voice.ts +7 -3
|
@@ -1,141 +1,45 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* A voice call's live levels, as React state at ~20fps.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* the
|
|
4
|
+
* `useVoiceRoom` writes the microphone level (from the recognizer) and the
|
|
5
|
+
* playback level (from the clip being spoken) into a `VoiceLevelSource`; this
|
|
6
|
+
* reads them on a clock so a wave can follow them without rendering once per
|
|
7
|
+
* audio buffer. It used to put Web Audio analysers on a LiveKit room's tracks;
|
|
8
|
+
* the signature kept its shape — `room` is now the level source `useVoiceRoom`
|
|
9
|
+
* returns under that name.
|
|
9
10
|
*/
|
|
10
11
|
|
|
11
|
-
import { useState, useEffect
|
|
12
|
-
import
|
|
12
|
+
import { useState, useEffect } from 'react';
|
|
13
|
+
import type { VoiceLevelSource } from '../lib/voice-levels';
|
|
13
14
|
|
|
14
|
-
|
|
15
|
-
function getRMS(analyser: AnalyserNode, buffer: Float32Array<ArrayBuffer>): number {
|
|
16
|
-
analyser.getFloatTimeDomainData(buffer);
|
|
17
|
-
let sum = 0;
|
|
18
|
-
for (let i = 0; i < buffer.length; i++) sum += buffer[i] * buffer[i];
|
|
19
|
-
return Math.min(1, Math.sqrt(sum / buffer.length) * 8);
|
|
20
|
-
}
|
|
15
|
+
const POLL_MS = 50;
|
|
21
16
|
|
|
22
|
-
export function useAudioLevelMonitor(room:
|
|
17
|
+
export function useAudioLevelMonitor(room: VoiceLevelSource | null, isConnected: boolean) {
|
|
23
18
|
const [captureLevel, setCaptureLevel] = useState(0);
|
|
24
19
|
const [playbackLevel, setPlaybackLevel] = useState(0);
|
|
25
|
-
const cleanupRef = useRef<(() => void) | null>(null);
|
|
26
20
|
|
|
27
21
|
useEffect(() => {
|
|
28
|
-
// Tear down previous monitors
|
|
29
|
-
cleanupRef.current?.();
|
|
30
|
-
cleanupRef.current = null;
|
|
31
|
-
|
|
32
22
|
if (!room || !isConnected) {
|
|
33
23
|
setCaptureLevel(0);
|
|
34
24
|
setPlaybackLevel(0);
|
|
35
|
-
return;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
// Guard: Web Audio API required
|
|
39
|
-
if (typeof AudioContext === 'undefined') return;
|
|
40
|
-
|
|
41
|
-
const audioCtx = new AudioContext();
|
|
42
|
-
// Resume AudioContext (may be suspended until user gesture on mobile/WebView)
|
|
43
|
-
if (audioCtx.state === 'suspended') {
|
|
44
|
-
audioCtx.resume().catch(() => {});
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
let localAnalyser: AnalyserNode | null = null;
|
|
48
|
-
let localBuffer: Float32Array<ArrayBuffer> | null = null;
|
|
49
|
-
const remoteAnalysers = new Map<string, { analyser: AnalyserNode; buffer: Float32Array<ArrayBuffer> }>();
|
|
50
|
-
let rafId = 0;
|
|
51
|
-
let disposed = false;
|
|
52
|
-
|
|
53
|
-
// ---------- Local mic ----------
|
|
54
|
-
const localTrack = room.localParticipant?.getTrackPublication(Track.Source.Microphone)?.track;
|
|
55
|
-
if (localTrack?.mediaStreamTrack) {
|
|
56
|
-
try {
|
|
57
|
-
const source = audioCtx.createMediaStreamSource(new MediaStream([localTrack.mediaStreamTrack]));
|
|
58
|
-
localAnalyser = audioCtx.createAnalyser();
|
|
59
|
-
localAnalyser.fftSize = 256;
|
|
60
|
-
source.connect(localAnalyser);
|
|
61
|
-
localBuffer = new Float32Array(localAnalyser.fftSize);
|
|
62
|
-
} catch {
|
|
63
|
-
// MediaStream creation can fail in some environments
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
// ---------- Remote agent audio (supports multiple tracks) ----------
|
|
68
|
-
const setupRemoteAnalyser = (mediaStreamTrack: MediaStreamTrack, key: string) => {
|
|
69
|
-
if (remoteAnalysers.has(key) || disposed) return;
|
|
70
|
-
try {
|
|
71
|
-
const source = audioCtx.createMediaStreamSource(new MediaStream([mediaStreamTrack]));
|
|
72
|
-
const analyser = audioCtx.createAnalyser();
|
|
73
|
-
analyser.fftSize = 256;
|
|
74
|
-
source.connect(analyser);
|
|
75
|
-
const buffer = new Float32Array(analyser.fftSize);
|
|
76
|
-
remoteAnalysers.set(key, { analyser, buffer });
|
|
77
|
-
} catch {
|
|
78
|
-
// Ignore errors
|
|
79
|
-
}
|
|
80
|
-
};
|
|
81
|
-
|
|
82
|
-
// Check already-subscribed remote tracks
|
|
83
|
-
for (const [, p] of room.remoteParticipants) {
|
|
84
|
-
for (const [, pub] of p.audioTrackPublications) {
|
|
85
|
-
if (pub.track?.mediaStreamTrack) {
|
|
86
|
-
setupRemoteAnalyser(pub.track.mediaStreamTrack, `${p.identity}-${pub.trackSid}`);
|
|
87
|
-
}
|
|
88
|
-
}
|
|
25
|
+
return undefined;
|
|
89
26
|
}
|
|
90
27
|
|
|
91
|
-
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
28
|
+
let dirty = true;
|
|
29
|
+
const unsubscribe = room.subscribe(() => {
|
|
30
|
+
dirty = true;
|
|
31
|
+
});
|
|
32
|
+
const timer = setInterval(() => {
|
|
33
|
+
if (!dirty) return;
|
|
34
|
+
dirty = false;
|
|
35
|
+
setCaptureLevel(room.capture);
|
|
36
|
+
setPlaybackLevel(room.playback);
|
|
37
|
+
}, POLL_MS);
|
|
38
|
+
|
|
39
|
+
return () => {
|
|
40
|
+
clearInterval(timer);
|
|
41
|
+
unsubscribe();
|
|
103
42
|
};
|
|
104
|
-
room.on(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);
|
|
105
|
-
|
|
106
|
-
// ---------- Poll loop (~20fps) ----------
|
|
107
|
-
let lastUpdate = 0;
|
|
108
|
-
const poll = (time: number) => {
|
|
109
|
-
if (disposed) return;
|
|
110
|
-
if (time - lastUpdate > 50) {
|
|
111
|
-
lastUpdate = time;
|
|
112
|
-
if (localAnalyser && localBuffer) {
|
|
113
|
-
setCaptureLevel(getRMS(localAnalyser, localBuffer));
|
|
114
|
-
}
|
|
115
|
-
// Take max RMS across all remote tracks (primary + cohost)
|
|
116
|
-
let maxRemoteLevel = 0;
|
|
117
|
-
for (const [, { analyser, buffer }] of remoteAnalysers) {
|
|
118
|
-
const level = getRMS(analyser, buffer);
|
|
119
|
-
if (level > maxRemoteLevel) maxRemoteLevel = level;
|
|
120
|
-
}
|
|
121
|
-
setPlaybackLevel(maxRemoteLevel);
|
|
122
|
-
}
|
|
123
|
-
rafId = requestAnimationFrame(poll);
|
|
124
|
-
};
|
|
125
|
-
rafId = requestAnimationFrame(poll);
|
|
126
|
-
|
|
127
|
-
// ---------- Cleanup ----------
|
|
128
|
-
const dispose = () => {
|
|
129
|
-
disposed = true;
|
|
130
|
-
cancelAnimationFrame(rafId);
|
|
131
|
-
room.off(RoomEvent.TrackSubscribed, onTrackSubscribed);
|
|
132
|
-
room.off(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);
|
|
133
|
-
remoteAnalysers.clear();
|
|
134
|
-
audioCtx.close().catch(() => {});
|
|
135
|
-
};
|
|
136
|
-
cleanupRef.current = dispose;
|
|
137
|
-
|
|
138
|
-
return dispose;
|
|
139
43
|
}, [room, isConnected]);
|
|
140
44
|
|
|
141
45
|
return { captureLevel, playbackLevel };
|
|
@@ -1,23 +1,22 @@
|
|
|
1
1
|
import { useState, useCallback, useRef, useEffect } from 'react';
|
|
2
|
-
import { errorMessage } from '../lib/utils';
|
|
3
|
-
import {
|
|
4
|
-
useAudioRecorder,
|
|
5
|
-
useAudioRecorderState,
|
|
6
|
-
RecordingPresets,
|
|
7
|
-
getRecordingPermissionsAsync,
|
|
8
|
-
setAudioModeAsync,
|
|
9
|
-
} from 'expo-audio';
|
|
10
|
-
import { useOxy } from '@oxy.so/services';
|
|
11
2
|
import { create } from 'zustand';
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
|
|
3
|
+
import {
|
|
4
|
+
isSpeechRecognitionAvailable,
|
|
5
|
+
requestSpeechRecognitionPermission,
|
|
6
|
+
startSpeechRecognition,
|
|
7
|
+
} from '../lib/speech-recognition';
|
|
8
|
+
import type { SpeechRecognitionSession } from '../lib/speech-recognition-types';
|
|
9
|
+
import { speechFailureMessage } from '../lib/speech-messages';
|
|
10
|
+
import { defaultSpeechLanguage } from '../lib/speech-language';
|
|
15
11
|
|
|
16
12
|
// ============== OPTIONS ==============
|
|
17
13
|
|
|
18
14
|
export interface UseSTTOptions {
|
|
19
|
-
|
|
20
|
-
|
|
15
|
+
/**
|
|
16
|
+
* Language to recognize, as a BCP-47 tag (`es-ES`). Defaults to the
|
|
17
|
+
* platform's own language. The Alia app passes its UI locale.
|
|
18
|
+
*/
|
|
19
|
+
lang?: string;
|
|
21
20
|
}
|
|
22
21
|
|
|
23
22
|
// ============== INLINE STT STORE ==============
|
|
@@ -40,169 +39,185 @@ export const useSTTStore = create<STTStoreState>((set) => ({
|
|
|
40
39
|
|
|
41
40
|
type STTState = 'idle' | 'recording' | 'transcribing';
|
|
42
41
|
|
|
43
|
-
|
|
44
|
-
|
|
42
|
+
/** How long `stopAndTranscribe` waits for the engine's final words. */
|
|
43
|
+
const FINAL_RESULT_TIMEOUT_MS = 2_000;
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* How many times one dictation may quietly re-open the recognizer.
|
|
47
|
+
*
|
|
48
|
+
* Browsers end a continuous session on their own — Chrome after a stretch of
|
|
49
|
+
* silence or about a minute of audio — and the person is still looking at a
|
|
50
|
+
* dictation bar that says it is listening. Re-opening keeps what was already
|
|
51
|
+
* heard and carries on. The cap is what stops an engine that ends the instant
|
|
52
|
+
* it starts from spinning.
|
|
53
|
+
*/
|
|
54
|
+
const MAX_SILENT_RESTARTS = 20;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Dictation, recognized on the device.
|
|
58
|
+
*
|
|
59
|
+
* The public shape is the one the recorder-and-upload version had —
|
|
60
|
+
* `startRecording`, `stopAndTranscribe`, `cancel`, the three states and
|
|
61
|
+
* `useSTTStore`'s metering — so the composer and the dictation bar did not
|
|
62
|
+
* change. What changed underneath: words arrive while the person speaks
|
|
63
|
+
* (`speech-recognition.ts` on web, `.native.ts` on iOS/Android), so
|
|
64
|
+
* `transcribing` is now the moment between "stop" and the engine's final
|
|
65
|
+
* result rather than an upload, and there is no transcription endpoint.
|
|
66
|
+
*/
|
|
45
67
|
export function useSpeechToText(options: UseSTTOptions = {}) {
|
|
46
|
-
const
|
|
68
|
+
const lang = options.lang ?? defaultSpeechLanguage();
|
|
47
69
|
|
|
48
70
|
const [state, setState] = useState<STTState>('idle');
|
|
49
71
|
const [error, setError] = useState<string | null>(null);
|
|
50
|
-
const
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
const
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const getToken = useCallback((): string | null => {
|
|
61
|
-
if (options.accessToken) return options.accessToken;
|
|
62
|
-
return oxyServices.httpService.getAccessToken();
|
|
63
|
-
}, [options.accessToken, oxyServices]);
|
|
64
|
-
|
|
65
|
-
// Push metering to store while recording (with epsilon guard to avoid thrashing subscribers)
|
|
72
|
+
const [isSupported] = useState(isSpeechRecognitionAvailable);
|
|
73
|
+
|
|
74
|
+
const stateRef = useRef<STTState>('idle');
|
|
75
|
+
const sessionRef = useRef<SpeechRecognitionSession | null>(null);
|
|
76
|
+
/** Text from sessions that already ended during this dictation. */
|
|
77
|
+
const earlierTextRef = useRef('');
|
|
78
|
+
/** Text from the live session. */
|
|
79
|
+
const currentTextRef = useRef('');
|
|
80
|
+
const restartsRef = useRef(0);
|
|
81
|
+
const endWaitersRef = useRef<Array<() => void>>([]);
|
|
66
82
|
const lastMeteringRef = useRef(0);
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
target = levelFromDbfs(recorderState.metering);
|
|
74
|
-
} else {
|
|
75
|
-
// Metering unavailable — simulate gentle activity
|
|
76
|
-
target = 0.12 + Math.random() * 0.18;
|
|
77
|
-
}
|
|
78
|
-
if (Math.abs(target - lastMeteringRef.current) >= 0.02) {
|
|
79
|
-
lastMeteringRef.current = target;
|
|
80
|
-
sttStore.getState().setMetering(target);
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
}, [state, recorderState.metering, recorderState.durationMillis]);
|
|
83
|
+
const mountedRef = useRef(true);
|
|
84
|
+
|
|
85
|
+
const setPhase = useCallback((next: STTState) => {
|
|
86
|
+
stateRef.current = next;
|
|
87
|
+
if (mountedRef.current) setState(next);
|
|
88
|
+
}, []);
|
|
84
89
|
|
|
85
90
|
// Sync recording state to store + reset metering on stop
|
|
86
91
|
useEffect(() => {
|
|
87
|
-
|
|
92
|
+
useSTTStore.getState().setRecording(state === 'recording');
|
|
88
93
|
if (state !== 'recording') {
|
|
89
94
|
lastMeteringRef.current = 0;
|
|
90
|
-
|
|
95
|
+
useSTTStore.getState().setMetering(0);
|
|
91
96
|
}
|
|
92
97
|
}, [state]);
|
|
93
98
|
|
|
99
|
+
const fullText = (): string =>
|
|
100
|
+
`${earlierTextRef.current} ${currentTextRef.current}`.replace(/\s+/g, ' ').trim();
|
|
101
|
+
|
|
102
|
+
const openSession = useCallback(() => {
|
|
103
|
+
currentTextRef.current = '';
|
|
104
|
+
let failed = false;
|
|
105
|
+
sessionRef.current = startSpeechRecognition(
|
|
106
|
+
{ lang },
|
|
107
|
+
{
|
|
108
|
+
onResult: ({ transcript }) => {
|
|
109
|
+
currentTextRef.current = transcript;
|
|
110
|
+
},
|
|
111
|
+
onLevel: (level) => {
|
|
112
|
+
if (stateRef.current !== 'recording') return;
|
|
113
|
+
// Epsilon guard: quiet should not thrash the store's subscribers.
|
|
114
|
+
if (Math.abs(level - lastMeteringRef.current) < 0.02) return;
|
|
115
|
+
lastMeteringRef.current = level;
|
|
116
|
+
useSTTStore.getState().setMetering(level);
|
|
117
|
+
},
|
|
118
|
+
onError: (failure) => {
|
|
119
|
+
const message = speechFailureMessage(failure);
|
|
120
|
+
if (message === null) return;
|
|
121
|
+
failed = true;
|
|
122
|
+
if (mountedRef.current) setError(message);
|
|
123
|
+
},
|
|
124
|
+
onEnd: () => {
|
|
125
|
+
sessionRef.current = null;
|
|
126
|
+
earlierTextRef.current = fullText();
|
|
127
|
+
currentTextRef.current = '';
|
|
128
|
+
const waiters = endWaitersRef.current;
|
|
129
|
+
endWaitersRef.current = [];
|
|
130
|
+
for (const resolve of waiters) resolve();
|
|
131
|
+
if (waiters.length > 0) return;
|
|
132
|
+
|
|
133
|
+
// The engine ended on its own while the person is still dictating.
|
|
134
|
+
if (stateRef.current !== 'recording') return;
|
|
135
|
+
if (failed || restartsRef.current >= MAX_SILENT_RESTARTS) {
|
|
136
|
+
setPhase('idle');
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
restartsRef.current += 1;
|
|
140
|
+
openSession();
|
|
141
|
+
},
|
|
142
|
+
},
|
|
143
|
+
);
|
|
144
|
+
}, [lang, setPhase]);
|
|
145
|
+
|
|
94
146
|
const startRecording = useCallback(async () => {
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
// getRecordingPermissionsAsync consults the Permissions API first (and
|
|
99
|
-
// only falls back to a real getUserMedia prompt when undetermined), so an
|
|
100
|
-
// already-granted permission never gets misreported as denied when the
|
|
101
|
-
// device itself is missing/busy — that failure surfaces from
|
|
102
|
-
// prepareToRecordAsync below with an accurate message.
|
|
103
|
-
const permStatus = await getRecordingPermissionsAsync();
|
|
104
|
-
if (!permStatus.granted) {
|
|
105
|
-
setError('Microphone permission required');
|
|
106
|
-
return;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
setState('recording');
|
|
110
|
-
|
|
111
|
-
await setAudioModeAsync({
|
|
112
|
-
allowsRecording: true,
|
|
113
|
-
playsInSilentMode: true,
|
|
114
|
-
});
|
|
147
|
+
if (stateRef.current !== 'idle') return;
|
|
148
|
+
setError(null);
|
|
115
149
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
} catch (e: unknown) {
|
|
120
|
-
const name = e instanceof Error ? e.name : '';
|
|
121
|
-
if (name === 'NotFoundError' || name === 'OverconstrainedError') {
|
|
122
|
-
setError('No microphone found — check your input devices');
|
|
123
|
-
} else if (name === 'NotAllowedError') {
|
|
124
|
-
setError('Microphone permission required');
|
|
125
|
-
} else if (name === 'NotReadableError') {
|
|
126
|
-
setError('Microphone is in use by another application');
|
|
127
|
-
} else {
|
|
128
|
-
setError('Failed to start recording');
|
|
129
|
-
}
|
|
130
|
-
setState('idle');
|
|
131
|
-
isRecordingRef.current = false;
|
|
150
|
+
if (!isSpeechRecognitionAvailable()) {
|
|
151
|
+
setError(speechFailureMessage({ code: 'unsupported' }));
|
|
152
|
+
return;
|
|
132
153
|
}
|
|
133
|
-
}, [audioRecorder]);
|
|
134
154
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
if (!uri) {
|
|
145
|
-
setState('idle');
|
|
146
|
-
return null;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
// Read audio file as base64
|
|
150
|
-
const response = await fetch(uri);
|
|
151
|
-
const blob = await response.blob();
|
|
152
|
-
// Detect actual MIME type (web records webm, native records m4a)
|
|
153
|
-
const detectedFormat = blob.type?.split(';')[0] || 'audio/m4a';
|
|
154
|
-
const base64 = await new Promise<string>((resolve, reject) => {
|
|
155
|
-
const reader = new FileReader();
|
|
156
|
-
reader.onload = () => {
|
|
157
|
-
const result = reader.result as string;
|
|
158
|
-
resolve(result.split(',')[1]);
|
|
159
|
-
};
|
|
160
|
-
reader.onerror = reject;
|
|
161
|
-
reader.readAsDataURL(blob);
|
|
162
|
-
});
|
|
155
|
+
// Claim the state before the first await so a double tap cannot open two.
|
|
156
|
+
setPhase('recording');
|
|
157
|
+
const refusal = await requestSpeechRecognitionPermission();
|
|
158
|
+
if ((stateRef.current as STTState) !== 'recording') return; // cancelled while asking
|
|
159
|
+
if (refusal !== null) {
|
|
160
|
+
setError(speechFailureMessage(refusal));
|
|
161
|
+
setPhase('idle');
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
163
164
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
165
|
+
earlierTextRef.current = '';
|
|
166
|
+
currentTextRef.current = '';
|
|
167
|
+
restartsRef.current = 0;
|
|
168
|
+
openSession();
|
|
169
|
+
}, [openSession, setPhase]);
|
|
170
|
+
|
|
171
|
+
const waitForEnd = useCallback((): Promise<void> => {
|
|
172
|
+
if (sessionRef.current === null) return Promise.resolve();
|
|
173
|
+
return new Promise<void>((resolve) => {
|
|
174
|
+
const timer = setTimeout(() => {
|
|
175
|
+
sessionRef.current?.abort();
|
|
176
|
+
resolve();
|
|
177
|
+
}, FINAL_RESULT_TIMEOUT_MS);
|
|
178
|
+
endWaitersRef.current.push(() => {
|
|
179
|
+
clearTimeout(timer);
|
|
180
|
+
resolve();
|
|
176
181
|
});
|
|
182
|
+
});
|
|
183
|
+
}, []);
|
|
177
184
|
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
return null;
|
|
191
|
-
}
|
|
192
|
-
}, [audioRecorder, getToken, apiUrl]);
|
|
185
|
+
const stopAndTranscribe = useCallback(async (): Promise<string | null> => {
|
|
186
|
+
if (stateRef.current !== 'recording') return null;
|
|
187
|
+
setPhase('transcribing');
|
|
188
|
+
const ended = waitForEnd();
|
|
189
|
+
sessionRef.current?.stop();
|
|
190
|
+
await ended;
|
|
191
|
+
const text = fullText();
|
|
192
|
+
earlierTextRef.current = '';
|
|
193
|
+
currentTextRef.current = '';
|
|
194
|
+
setPhase('idle');
|
|
195
|
+
return text === '' ? null : text;
|
|
196
|
+
}, [setPhase, waitForEnd]);
|
|
193
197
|
|
|
194
198
|
const cancel = useCallback(() => {
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
199
|
+
const session = sessionRef.current;
|
|
200
|
+
setPhase('idle');
|
|
201
|
+
earlierTextRef.current = '';
|
|
202
|
+
currentTextRef.current = '';
|
|
203
|
+
session?.abort();
|
|
200
204
|
setError(null);
|
|
201
|
-
}, [
|
|
205
|
+
}, [setPhase]);
|
|
206
|
+
|
|
207
|
+
useEffect(() => {
|
|
208
|
+
mountedRef.current = true;
|
|
209
|
+
return () => {
|
|
210
|
+
mountedRef.current = false;
|
|
211
|
+
stateRef.current = 'idle';
|
|
212
|
+
sessionRef.current?.abort();
|
|
213
|
+
};
|
|
214
|
+
}, []);
|
|
202
215
|
|
|
203
216
|
return {
|
|
204
217
|
state,
|
|
205
218
|
error,
|
|
219
|
+
/** False where the platform cannot recognize speech at all (Firefox). */
|
|
220
|
+
isSupported,
|
|
206
221
|
startRecording,
|
|
207
222
|
stopAndTranscribe,
|
|
208
223
|
cancel,
|
package/src/hooks/useTTS.ts
CHANGED
|
@@ -6,6 +6,7 @@ import { errorMessage } from '../lib/utils';
|
|
|
6
6
|
import { create } from 'zustand';
|
|
7
7
|
import { createAudioLevelMeter } from '../lib/audio-level';
|
|
8
8
|
import { PREFERRED_VOICE_MODEL_ID } from '../lib/config';
|
|
9
|
+
import { requestSpeechClip } from '../lib/speech-synthesis';
|
|
9
10
|
|
|
10
11
|
const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
|
|
11
12
|
|
|
@@ -105,6 +106,7 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
105
106
|
const reset = useTTSStore((s) => s.reset);
|
|
106
107
|
|
|
107
108
|
const playerRef = useRef<AudioPlayer | null>(null);
|
|
109
|
+
const requestRef = useRef<AbortController | null>(null);
|
|
108
110
|
|
|
109
111
|
// ============== AUTH ==============
|
|
110
112
|
|
|
@@ -114,7 +116,7 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
114
116
|
}, [options.accessToken, oxyServices]);
|
|
115
117
|
|
|
116
118
|
const getTTSVoice = useCallback(() => {
|
|
117
|
-
return voicePref === 'male' ? '
|
|
119
|
+
return voicePref === 'male' ? 'male' : 'female';
|
|
118
120
|
}, [voicePref]);
|
|
119
121
|
|
|
120
122
|
const getTTSSpeed = useCallback(() => {
|
|
@@ -133,6 +135,8 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
133
135
|
}, []);
|
|
134
136
|
|
|
135
137
|
const stop = useCallback(() => {
|
|
138
|
+
requestRef.current?.abort();
|
|
139
|
+
requestRef.current = null;
|
|
136
140
|
releasePlayer();
|
|
137
141
|
reset();
|
|
138
142
|
}, [reset, releasePlayer]);
|
|
@@ -150,6 +154,7 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
150
154
|
const playFromUrl = useCallback((
|
|
151
155
|
audioUrl: string,
|
|
152
156
|
_messageId: string,
|
|
157
|
+
signal: AbortSignal,
|
|
153
158
|
onUnplayable?: () => void,
|
|
154
159
|
) => {
|
|
155
160
|
releasePlayer();
|
|
@@ -157,6 +162,7 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
157
162
|
(async () => {
|
|
158
163
|
try {
|
|
159
164
|
const { createAudioPlayer } = await import('expo-audio');
|
|
165
|
+
if (signal.aborted) return;
|
|
160
166
|
/**
|
|
161
167
|
* `crossOrigin` is web-only and ignored elsewhere, and it is what makes
|
|
162
168
|
* the waveform readable there: a browser will not let an `AnalyserNode`
|
|
@@ -199,6 +205,7 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
199
205
|
});
|
|
200
206
|
|
|
201
207
|
player.addListener('playbackStatusUpdate', (status: AudioStatus) => {
|
|
208
|
+
if (signal.aborted || playerRef.current !== player) return;
|
|
202
209
|
if (status.error) {
|
|
203
210
|
releasePlayer();
|
|
204
211
|
if (onUnplayable) {
|
|
@@ -217,6 +224,7 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
217
224
|
player.play();
|
|
218
225
|
setPlaybackState('playing');
|
|
219
226
|
} catch {
|
|
227
|
+
if (signal.aborted) return;
|
|
220
228
|
if (onUnplayable) {
|
|
221
229
|
onUnplayable();
|
|
222
230
|
return;
|
|
@@ -256,6 +264,8 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
256
264
|
stop();
|
|
257
265
|
}
|
|
258
266
|
|
|
267
|
+
const controller = new AbortController();
|
|
268
|
+
requestRef.current = controller;
|
|
259
269
|
try {
|
|
260
270
|
setActiveMessage(messageId);
|
|
261
271
|
setPlaybackState('loading');
|
|
@@ -274,32 +284,18 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
274
284
|
throw new Error('Not authenticated');
|
|
275
285
|
}
|
|
276
286
|
|
|
277
|
-
const
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
speed: getTTSSpeed(),
|
|
288
|
-
conversationId,
|
|
289
|
-
messageId,
|
|
290
|
-
}),
|
|
287
|
+
const audioUrl = await requestSpeechClip({
|
|
288
|
+
apiUrl,
|
|
289
|
+
token,
|
|
290
|
+
model: voiceModel,
|
|
291
|
+
input: text,
|
|
292
|
+
voice: getTTSVoice(),
|
|
293
|
+
speed: getTTSSpeed(),
|
|
294
|
+
conversationId,
|
|
295
|
+
messageId,
|
|
296
|
+
signal: controller.signal,
|
|
291
297
|
});
|
|
292
|
-
|
|
293
|
-
if (!response.ok) {
|
|
294
|
-
if (response.status === 504) {
|
|
295
|
-
throw new Error('Request timed out — please try again');
|
|
296
|
-
}
|
|
297
|
-
const errData = await response.json().catch(() => ({}));
|
|
298
|
-
throw new Error(errData.error?.message || errData.error || 'TTS failed');
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
const data = await response.json();
|
|
302
|
-
playFromUrl(data.audioUrl, messageId);
|
|
298
|
+
playFromUrl(audioUrl, messageId, controller.signal);
|
|
303
299
|
};
|
|
304
300
|
|
|
305
301
|
/**
|
|
@@ -311,22 +307,26 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
311
307
|
* forever, and the person notices nothing.
|
|
312
308
|
*/
|
|
313
309
|
if (audioUrl) {
|
|
314
|
-
playFromUrl(audioUrl, messageId, () => {
|
|
315
|
-
void synthesize().catch((e: unknown) =>
|
|
310
|
+
playFromUrl(audioUrl, messageId, controller.signal, () => {
|
|
311
|
+
void synthesize().catch((e: unknown) => {
|
|
312
|
+
if (!controller.signal.aborted) setError(errorMessage(e, 'Failed to read aloud'));
|
|
313
|
+
});
|
|
316
314
|
});
|
|
317
315
|
return;
|
|
318
316
|
}
|
|
319
317
|
|
|
320
318
|
await synthesize();
|
|
321
319
|
} catch (e: unknown) {
|
|
320
|
+
if (controller.signal.aborted) return;
|
|
322
321
|
console.error('[TTS] Error:', e);
|
|
323
322
|
setError(errorMessage(e, 'Failed to read aloud'));
|
|
324
323
|
}
|
|
325
|
-
}, [getToken, apiUrl, getTTSVoice, getTTSSpeed, stop, playFromUrl, setActiveMessage, setPlaybackState, setError]);
|
|
324
|
+
}, [getToken, apiUrl, voiceModel, getTTSVoice, getTTSSpeed, stop, playFromUrl, setActiveMessage, setPlaybackState, setError]);
|
|
326
325
|
|
|
327
326
|
// Cleanup on unmount
|
|
328
327
|
useEffect(() => {
|
|
329
328
|
return () => {
|
|
329
|
+
requestRef.current?.abort();
|
|
330
330
|
releasePlayer();
|
|
331
331
|
};
|
|
332
332
|
}, []);
|