@alia.onl/sdk 7.2.9 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +142 -6
- package/README.md +242 -0
- package/package.json +8 -12
- package/src/components/AliaChatContent.tsx +0 -1
- package/src/components/AliaChatMessageList.tsx +0 -4
- package/src/components/IdentityMark.tsx +3 -0
- package/src/components/Markdown.tsx +89 -8
- package/src/components/ResearchProgressCard.tsx +3 -0
- package/src/components/ui/prompt-input/mic-button.tsx +2 -3
- package/src/components/ui/prompt-input/prompt-input.tsx +1 -4
- package/src/components/voice/VoiceControls.tsx +48 -63
- package/src/components/voice/VoiceSession.tsx +6 -19
- package/src/hooks/useAliaChat.ts +24 -10
- package/src/hooks/useAudioLevelMonitor.ts +26 -122
- package/src/hooks/useSpeechToText.ts +196 -147
- package/src/hooks/useTTS.ts +33 -38
- package/src/hooks/useVoiceRoom.ts +581 -416
- package/src/index.ts +6 -19
- package/src/lib/audio-level.ts +3 -2
- package/src/lib/catalogue.ts +116 -103
- package/src/lib/chat-stream.ts +0 -5
- package/src/lib/chat-transport.ts +14 -2
- package/src/lib/speech-language.ts +20 -0
- package/src/lib/speech-messages.ts +76 -0
- package/src/lib/speech-recognition-types.ts +108 -0
- package/src/lib/speech-recognition.native.ts +226 -0
- package/src/lib/speech-recognition.ts +288 -0
- package/src/lib/speech-synthesis.ts +122 -0
- package/src/lib/voice-levels.ts +60 -0
- package/src/lib/voice-text.ts +192 -0
- package/src/lib/voice-turn.ts +86 -0
- package/src/types.ts +3 -3
- package/src/voice.ts +11 -3
- package/src/lib/config.ts +0 -36
|
@@ -49,8 +49,6 @@ export type PromptInputProps = {
|
|
|
49
49
|
onSuggestionUsed?: (suggestionId: string) => void;
|
|
50
50
|
// Injectable error handler (replaces toast)
|
|
51
51
|
onError?: (message: string) => void;
|
|
52
|
-
// Mic button API URL
|
|
53
|
-
apiUrl?: string;
|
|
54
52
|
} & Omit<React.ComponentProps<typeof View>, "children">;
|
|
55
53
|
|
|
56
54
|
export function PromptInput({
|
|
@@ -77,7 +75,6 @@ export function PromptInput({
|
|
|
77
75
|
useSuggestions,
|
|
78
76
|
onSuggestionUsed,
|
|
79
77
|
onError,
|
|
80
|
-
apiUrl,
|
|
81
78
|
...props
|
|
82
79
|
}: PromptInputProps) {
|
|
83
80
|
const [internalValue, setInternalValue] = useState(value || "");
|
|
@@ -182,7 +179,7 @@ export function PromptInput({
|
|
|
182
179
|
{actionsLeft ?? <PromptInputAddMenu />}
|
|
183
180
|
</View>
|
|
184
181
|
<View className="flex-row items-center gap-1.5">
|
|
185
|
-
<PromptInputMicButton
|
|
182
|
+
<PromptInputMicButton />
|
|
186
183
|
<PromptInputSubmitButton
|
|
187
184
|
isLoading={isLoading}
|
|
188
185
|
onStop={onStop}
|
|
@@ -1,46 +1,63 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Bottom voice control bar that replaces
|
|
3
|
-
* Contains status text
|
|
2
|
+
* Bottom voice control bar that replaces the composer when voice mode is active.
|
|
3
|
+
* Contains status text and mute/end buttons.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { View, Text, Pressable, ActivityIndicator, StyleSheet } from 'react-native';
|
|
7
7
|
import Mic from 'lucide-react-native/icons/mic';
|
|
8
8
|
import MicOff from 'lucide-react-native/icons/mic-off';
|
|
9
9
|
import PhoneOff from 'lucide-react-native/icons/phone-off';
|
|
10
|
-
import Users from 'lucide-react-native/icons/users';
|
|
11
10
|
import type { RoomState, AgentState } from '../../types';
|
|
12
11
|
|
|
13
|
-
|
|
12
|
+
/** Every word the bar shows or announces. An app in another language passes its own. */
|
|
13
|
+
export interface VoiceControlsLabels {
|
|
14
|
+
connecting: string;
|
|
15
|
+
connected: string;
|
|
16
|
+
listening: string;
|
|
17
|
+
muted: string;
|
|
18
|
+
thinking: string;
|
|
19
|
+
speaking: string;
|
|
20
|
+
mute: string;
|
|
21
|
+
unmute: string;
|
|
22
|
+
end: string;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export const VOICE_CONTROLS_LABELS: VoiceControlsLabels = {
|
|
26
|
+
connecting: 'Connecting...',
|
|
27
|
+
connected: 'Connected',
|
|
28
|
+
listening: 'Listening...',
|
|
29
|
+
muted: 'Muted',
|
|
30
|
+
thinking: 'Thinking...',
|
|
31
|
+
speaking: 'Speaking...',
|
|
32
|
+
mute: 'Mute',
|
|
33
|
+
unmute: 'Unmute',
|
|
34
|
+
end: 'End',
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
export interface VoiceControlsProps {
|
|
14
38
|
roomState: RoomState;
|
|
15
39
|
agentState: AgentState;
|
|
16
40
|
isMuted: boolean;
|
|
17
|
-
cohostActive: boolean;
|
|
18
|
-
currentSpeaker: 'primary' | 'cohost' | 'user' | null;
|
|
19
|
-
roundComplete: boolean;
|
|
20
41
|
onToggleMute: () => void;
|
|
21
|
-
onEnableCohost: () => void;
|
|
22
|
-
onDisableCohost: () => void;
|
|
23
|
-
onContinueCohost: () => void;
|
|
24
42
|
onEnd: () => void;
|
|
25
43
|
/** Override theme primary color (resolved hex/hsl value, not CSS var) */
|
|
26
44
|
primaryColor?: string;
|
|
45
|
+
/** Overrides for any of {@link VOICE_CONTROLS_LABELS}. */
|
|
46
|
+
labels?: Partial<VoiceControlsLabels>;
|
|
27
47
|
}
|
|
28
48
|
|
|
29
49
|
function getStatusText(
|
|
30
50
|
roomState: RoomState,
|
|
31
51
|
agentState: AgentState,
|
|
32
52
|
isMuted: boolean,
|
|
33
|
-
|
|
34
|
-
currentSpeaker: string | null,
|
|
53
|
+
labels: VoiceControlsLabels,
|
|
35
54
|
): string {
|
|
36
|
-
if (roomState === 'connecting') return
|
|
55
|
+
if (roomState === 'connecting') return labels.connecting;
|
|
37
56
|
if (roomState === 'connected') {
|
|
38
|
-
if (
|
|
39
|
-
if (
|
|
40
|
-
if (agentState === '
|
|
41
|
-
|
|
42
|
-
if (agentState === 'speaking') return 'Speaking...';
|
|
43
|
-
return 'Connected';
|
|
57
|
+
if (agentState === 'listening') return isMuted ? labels.muted : labels.listening;
|
|
58
|
+
if (agentState === 'thinking') return labels.thinking;
|
|
59
|
+
if (agentState === 'speaking') return labels.speaking;
|
|
60
|
+
return labels.connected;
|
|
44
61
|
}
|
|
45
62
|
return '';
|
|
46
63
|
}
|
|
@@ -49,43 +66,31 @@ export function VoiceControls({
|
|
|
49
66
|
roomState,
|
|
50
67
|
agentState,
|
|
51
68
|
isMuted,
|
|
52
|
-
cohostActive,
|
|
53
|
-
currentSpeaker,
|
|
54
|
-
roundComplete,
|
|
55
69
|
onToggleMute,
|
|
56
|
-
onEnableCohost,
|
|
57
|
-
onDisableCohost,
|
|
58
|
-
onContinueCohost,
|
|
59
70
|
onEnd,
|
|
60
71
|
primaryColor,
|
|
72
|
+
labels: labelOverrides,
|
|
61
73
|
}: VoiceControlsProps) {
|
|
62
|
-
const
|
|
74
|
+
const labels = { ...VOICE_CONTROLS_LABELS, ...labelOverrides };
|
|
75
|
+
const statusText = getStatusText(roomState, agentState, isMuted, labels);
|
|
63
76
|
|
|
64
77
|
return (
|
|
65
78
|
<View style={styles.container}>
|
|
66
79
|
{statusText ? (
|
|
67
|
-
<Text className="text-lg font-medium mb-4 text-foreground">
|
|
80
|
+
<Text className="text-lg font-medium mb-4 text-foreground" accessibilityLiveRegion="polite" aria-live="polite">
|
|
68
81
|
{statusText}
|
|
69
82
|
</Text>
|
|
70
83
|
) : null}
|
|
71
84
|
|
|
72
|
-
{roundComplete && (
|
|
73
|
-
<Pressable
|
|
74
|
-
onPress={onContinueCohost}
|
|
75
|
-
className="bg-primary/30"
|
|
76
|
-
style={[styles.continueButton, primaryColor ? { backgroundColor: primaryColor + '4D' } : undefined]}
|
|
77
|
-
>
|
|
78
|
-
<Text className="text-sm font-medium text-primary" style={primaryColor ? { color: primaryColor } : undefined}>
|
|
79
|
-
Continue conversation
|
|
80
|
-
</Text>
|
|
81
|
-
</Pressable>
|
|
82
|
-
)}
|
|
83
|
-
|
|
84
85
|
{roomState === 'connected' && (
|
|
85
86
|
<View style={styles.buttonRow}>
|
|
86
87
|
<View style={styles.buttonWrapper}>
|
|
87
88
|
<Pressable
|
|
88
89
|
onPress={onToggleMute}
|
|
90
|
+
accessibilityRole="button"
|
|
91
|
+
accessibilityLabel={labels.mute}
|
|
92
|
+
accessibilityState={{ selected: isMuted }}
|
|
93
|
+
aria-pressed={isMuted}
|
|
89
94
|
className={isMuted ? undefined : 'bg-muted'}
|
|
90
95
|
style={[styles.button, isMuted ? { backgroundColor: '#ef4444' } : undefined]}
|
|
91
96
|
>
|
|
@@ -96,34 +101,20 @@ export function VoiceControls({
|
|
|
96
101
|
)}
|
|
97
102
|
</Pressable>
|
|
98
103
|
<Text className="text-xs text-muted-foreground">
|
|
99
|
-
{isMuted ?
|
|
100
|
-
</Text>
|
|
101
|
-
</View>
|
|
102
|
-
|
|
103
|
-
<View style={styles.buttonWrapper}>
|
|
104
|
-
<Pressable
|
|
105
|
-
onPress={cohostActive ? onDisableCohost : onEnableCohost}
|
|
106
|
-
className={cohostActive ? 'bg-primary' : 'bg-muted'}
|
|
107
|
-
style={[
|
|
108
|
-
styles.button,
|
|
109
|
-
cohostActive && primaryColor ? { backgroundColor: primaryColor } : undefined,
|
|
110
|
-
]}
|
|
111
|
-
>
|
|
112
|
-
<Users size={24} color="white" />
|
|
113
|
-
</Pressable>
|
|
114
|
-
<Text className="text-xs text-muted-foreground">
|
|
115
|
-
{cohostActive ? 'Solo' : 'Cohost'}
|
|
104
|
+
{isMuted ? labels.unmute : labels.mute}
|
|
116
105
|
</Text>
|
|
117
106
|
</View>
|
|
118
107
|
|
|
119
108
|
<View style={styles.buttonWrapper}>
|
|
120
109
|
<Pressable
|
|
121
110
|
onPress={onEnd}
|
|
111
|
+
accessibilityRole="button"
|
|
112
|
+
accessibilityLabel={labels.end}
|
|
122
113
|
style={[styles.button, { backgroundColor: '#ef4444' }]}
|
|
123
114
|
>
|
|
124
115
|
<PhoneOff size={24} color="white" />
|
|
125
116
|
</Pressable>
|
|
126
|
-
<Text className="text-xs text-muted-foreground">
|
|
117
|
+
<Text className="text-xs text-muted-foreground">{labels.end}</Text>
|
|
127
118
|
</View>
|
|
128
119
|
</View>
|
|
129
120
|
)}
|
|
@@ -157,10 +148,4 @@ const styles = StyleSheet.create({
|
|
|
157
148
|
alignItems: 'center',
|
|
158
149
|
justifyContent: 'center',
|
|
159
150
|
},
|
|
160
|
-
continueButton: {
|
|
161
|
-
marginBottom: 16,
|
|
162
|
-
paddingHorizontal: 20,
|
|
163
|
-
paddingVertical: 8,
|
|
164
|
-
borderRadius: 24,
|
|
165
|
-
},
|
|
166
151
|
});
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* A live
|
|
2
|
+
* A live voice call, mounted only while voice mode is on.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* which is the whole reason the `@alia.onl/sdk/voice` entry exists.
|
|
4
|
+
* The chat shell receives it as the `voiceSession` prop rather than importing
|
|
5
|
+
* it, so a text-only consumer never compiles the voice loop, speech playback or
|
|
6
|
+
* the call UI — which is the reason the `@alia.onl/sdk/voice` entry exists.
|
|
8
7
|
*
|
|
9
|
-
* Mounting starts the call; unmounting ends it (`useVoiceRoom`
|
|
10
|
-
*
|
|
8
|
+
* Mounting starts the call; unmounting ends it (`useVoiceRoom` stops listening
|
|
9
|
+
* and speaking on unmount), so the shell only has to decide whether to render it.
|
|
11
10
|
*/
|
|
12
11
|
|
|
13
12
|
import { useEffect, useRef } from 'react';
|
|
@@ -25,15 +24,9 @@ export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps
|
|
|
25
24
|
isMuted,
|
|
26
25
|
error,
|
|
27
26
|
messages,
|
|
28
|
-
cohostActive,
|
|
29
|
-
currentSpeaker,
|
|
30
|
-
roundComplete,
|
|
31
27
|
isConnected,
|
|
32
28
|
connect,
|
|
33
29
|
toggleMute,
|
|
34
|
-
enableCohost,
|
|
35
|
-
disableCohost,
|
|
36
|
-
continueCohost,
|
|
37
30
|
} = useVoiceRoom({ apiUrl });
|
|
38
31
|
|
|
39
32
|
const { captureLevel, playbackLevel } = useAudioLevelMonitor(room, isConnected);
|
|
@@ -75,13 +68,7 @@ export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps
|
|
|
75
68
|
roomState={roomState}
|
|
76
69
|
agentState={agentState}
|
|
77
70
|
isMuted={isMuted}
|
|
78
|
-
cohostActive={cohostActive}
|
|
79
|
-
currentSpeaker={currentSpeaker}
|
|
80
|
-
roundComplete={roundComplete}
|
|
81
71
|
onToggleMute={toggleMute}
|
|
82
|
-
onEnableCohost={enableCohost}
|
|
83
|
-
onDisableCohost={disableCohost}
|
|
84
|
-
onContinueCohost={continueCohost}
|
|
85
72
|
onEnd={onEnd}
|
|
86
73
|
/>
|
|
87
74
|
);
|
package/src/hooks/useAliaChat.ts
CHANGED
|
@@ -3,22 +3,36 @@ import { useOxy } from '@oxy.so/services';
|
|
|
3
3
|
import type { ChatMessage, ToolInvocation } from '../types';
|
|
4
4
|
import { getTextFromContent } from '../lib/content-utils';
|
|
5
5
|
import { resolveModelId } from '../lib/catalogue';
|
|
6
|
-
import { PREFERRED_CHAT_MODEL_ID } from '../lib/config';
|
|
7
6
|
import { streamAliaChat } from '../lib/chat-transport';
|
|
8
7
|
import type { AliaChatStreamEvent } from '../lib/chat-stream';
|
|
9
8
|
|
|
10
9
|
const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
|
|
11
10
|
|
|
12
11
|
export interface UseAliaChatOptions {
|
|
13
|
-
/**
|
|
12
|
+
/**
|
|
13
|
+
* Base URL the hook sends its requests to. Defaults to
|
|
14
|
+
* `EXPO_PUBLIC_ALIA_API_URL`, then `https://api.alia.onl`.
|
|
15
|
+
*
|
|
16
|
+
* The hook appends its own paths — `POST /v1/chat/completions` for a turn and
|
|
17
|
+
* `GET /catalogue` for the model check — so the value is a host, not a route.
|
|
18
|
+
*
|
|
19
|
+
* Point this at YOUR OWN backend to use the SDK from an origin Alia does not
|
|
20
|
+
* enumerate: Alia's product route, `POST /alia/chat`, answers CORS preflights
|
|
21
|
+
* only for Alia's own origins, while a server-to-server call carries no
|
|
22
|
+
* `Origin` header and is answered on its credential alone. Your backend
|
|
23
|
+
* receives the SDK's request, forwards the `Authorization` header it carries
|
|
24
|
+
* (the signed-in user's Oxy access token) to `/alia/chat`, and streams the
|
|
25
|
+
* SSE body back unchanged. The README's *consumer-backend path* section has
|
|
26
|
+
* the minimal relay; issue #244 records why this is the supported shape today.
|
|
27
|
+
*/
|
|
14
28
|
apiUrl?: string;
|
|
15
29
|
/**
|
|
16
|
-
*
|
|
30
|
+
* The model to answer with, as `publisher/model` from `GET /catalogue`.
|
|
17
31
|
*
|
|
18
|
-
* Optional, and checked against
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
32
|
+
* Optional, and checked against the catalogue before a request carries it: an
|
|
33
|
+
* identifier the server no longer offers is not sent, so a retirement does not
|
|
34
|
+
* turn into a 400 inside a consumer's app. Omitted, the request carries no
|
|
35
|
+
* `model` and the server's default model answers.
|
|
22
36
|
*/
|
|
23
37
|
model?: string;
|
|
24
38
|
/** App context injected as system message so Alia knows which app the user is in. */
|
|
@@ -100,7 +114,7 @@ function waitWithAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T>
|
|
|
100
114
|
export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn {
|
|
101
115
|
const {
|
|
102
116
|
apiUrl = API_URL,
|
|
103
|
-
model
|
|
117
|
+
model,
|
|
104
118
|
clientContext,
|
|
105
119
|
accessToken: accessTokenProp,
|
|
106
120
|
} = options;
|
|
@@ -221,7 +235,7 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
|
|
|
221
235
|
const trimmed = text.trim();
|
|
222
236
|
if (!trimmed) return;
|
|
223
237
|
|
|
224
|
-
const activeToken = oxyServices.
|
|
238
|
+
const activeToken = oxyServices.session.accessToken;
|
|
225
239
|
if (accessTokenProp !== undefined && accessTokenProp !== activeToken) {
|
|
226
240
|
setError('The supplied token is not the active Oxy session.');
|
|
227
241
|
return;
|
|
@@ -362,7 +376,7 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
|
|
|
362
376
|
|
|
363
377
|
try {
|
|
364
378
|
const effectiveModel = await waitWithAbort(
|
|
365
|
-
resolveModelId(apiUrl, model
|
|
379
|
+
resolveModelId(apiUrl, model),
|
|
366
380
|
controller.signal,
|
|
367
381
|
);
|
|
368
382
|
await streamAliaChat(
|
|
@@ -1,141 +1,45 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* A voice call's live levels, as React state at ~20fps.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* the
|
|
4
|
+
* `useVoiceRoom` writes the microphone level (from the recognizer) and the
|
|
5
|
+
* playback level (from the clip being spoken) into a `VoiceLevelSource`; this
|
|
6
|
+
* reads them on a clock so a wave can follow them without rendering once per
|
|
7
|
+
* audio buffer. It used to put Web Audio analysers on a LiveKit room's tracks;
|
|
8
|
+
* the signature kept its shape — `room` is now the level source `useVoiceRoom`
|
|
9
|
+
* returns under that name.
|
|
9
10
|
*/
|
|
10
11
|
|
|
11
|
-
import { useState, useEffect
|
|
12
|
-
import
|
|
12
|
+
import { useState, useEffect } from 'react';
|
|
13
|
+
import type { VoiceLevelSource } from '../lib/voice-levels';
|
|
13
14
|
|
|
14
|
-
|
|
15
|
-
function getRMS(analyser: AnalyserNode, buffer: Float32Array<ArrayBuffer>): number {
|
|
16
|
-
analyser.getFloatTimeDomainData(buffer);
|
|
17
|
-
let sum = 0;
|
|
18
|
-
for (let i = 0; i < buffer.length; i++) sum += buffer[i] * buffer[i];
|
|
19
|
-
return Math.min(1, Math.sqrt(sum / buffer.length) * 8);
|
|
20
|
-
}
|
|
15
|
+
const POLL_MS = 50;
|
|
21
16
|
|
|
22
|
-
export function useAudioLevelMonitor(room:
|
|
17
|
+
export function useAudioLevelMonitor(room: VoiceLevelSource | null, isConnected: boolean) {
|
|
23
18
|
const [captureLevel, setCaptureLevel] = useState(0);
|
|
24
19
|
const [playbackLevel, setPlaybackLevel] = useState(0);
|
|
25
|
-
const cleanupRef = useRef<(() => void) | null>(null);
|
|
26
20
|
|
|
27
21
|
useEffect(() => {
|
|
28
|
-
// Tear down previous monitors
|
|
29
|
-
cleanupRef.current?.();
|
|
30
|
-
cleanupRef.current = null;
|
|
31
|
-
|
|
32
22
|
if (!room || !isConnected) {
|
|
33
23
|
setCaptureLevel(0);
|
|
34
24
|
setPlaybackLevel(0);
|
|
35
|
-
return;
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
// Guard: Web Audio API required
|
|
39
|
-
if (typeof AudioContext === 'undefined') return;
|
|
40
|
-
|
|
41
|
-
const audioCtx = new AudioContext();
|
|
42
|
-
// Resume AudioContext (may be suspended until user gesture on mobile/WebView)
|
|
43
|
-
if (audioCtx.state === 'suspended') {
|
|
44
|
-
audioCtx.resume().catch(() => {});
|
|
45
|
-
}
|
|
46
|
-
|
|
47
|
-
let localAnalyser: AnalyserNode | null = null;
|
|
48
|
-
let localBuffer: Float32Array<ArrayBuffer> | null = null;
|
|
49
|
-
const remoteAnalysers = new Map<string, { analyser: AnalyserNode; buffer: Float32Array<ArrayBuffer> }>();
|
|
50
|
-
let rafId = 0;
|
|
51
|
-
let disposed = false;
|
|
52
|
-
|
|
53
|
-
// ---------- Local mic ----------
|
|
54
|
-
const localTrack = room.localParticipant?.getTrackPublication(Track.Source.Microphone)?.track;
|
|
55
|
-
if (localTrack?.mediaStreamTrack) {
|
|
56
|
-
try {
|
|
57
|
-
const source = audioCtx.createMediaStreamSource(new MediaStream([localTrack.mediaStreamTrack]));
|
|
58
|
-
localAnalyser = audioCtx.createAnalyser();
|
|
59
|
-
localAnalyser.fftSize = 256;
|
|
60
|
-
source.connect(localAnalyser);
|
|
61
|
-
localBuffer = new Float32Array(localAnalyser.fftSize);
|
|
62
|
-
} catch {
|
|
63
|
-
// MediaStream creation can fail in some environments
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
// ---------- Remote agent audio (supports multiple tracks) ----------
|
|
68
|
-
const setupRemoteAnalyser = (mediaStreamTrack: MediaStreamTrack, key: string) => {
|
|
69
|
-
if (remoteAnalysers.has(key) || disposed) return;
|
|
70
|
-
try {
|
|
71
|
-
const source = audioCtx.createMediaStreamSource(new MediaStream([mediaStreamTrack]));
|
|
72
|
-
const analyser = audioCtx.createAnalyser();
|
|
73
|
-
analyser.fftSize = 256;
|
|
74
|
-
source.connect(analyser);
|
|
75
|
-
const buffer = new Float32Array(analyser.fftSize);
|
|
76
|
-
remoteAnalysers.set(key, { analyser, buffer });
|
|
77
|
-
} catch {
|
|
78
|
-
// Ignore errors
|
|
79
|
-
}
|
|
80
|
-
};
|
|
81
|
-
|
|
82
|
-
// Check already-subscribed remote tracks
|
|
83
|
-
for (const [, p] of room.remoteParticipants) {
|
|
84
|
-
for (const [, pub] of p.audioTrackPublications) {
|
|
85
|
-
if (pub.track?.mediaStreamTrack) {
|
|
86
|
-
setupRemoteAnalyser(pub.track.mediaStreamTrack, `${p.identity}-${pub.trackSid}`);
|
|
87
|
-
}
|
|
88
|
-
}
|
|
25
|
+
return undefined;
|
|
89
26
|
}
|
|
90
27
|
|
|
91
|
-
|
|
92
|
-
const
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
28
|
+
let dirty = true;
|
|
29
|
+
const unsubscribe = room.subscribe(() => {
|
|
30
|
+
dirty = true;
|
|
31
|
+
});
|
|
32
|
+
const timer = setInterval(() => {
|
|
33
|
+
if (!dirty) return;
|
|
34
|
+
dirty = false;
|
|
35
|
+
setCaptureLevel(room.capture);
|
|
36
|
+
setPlaybackLevel(room.playback);
|
|
37
|
+
}, POLL_MS);
|
|
38
|
+
|
|
39
|
+
return () => {
|
|
40
|
+
clearInterval(timer);
|
|
41
|
+
unsubscribe();
|
|
103
42
|
};
|
|
104
|
-
room.on(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);
|
|
105
|
-
|
|
106
|
-
// ---------- Poll loop (~20fps) ----------
|
|
107
|
-
let lastUpdate = 0;
|
|
108
|
-
const poll = (time: number) => {
|
|
109
|
-
if (disposed) return;
|
|
110
|
-
if (time - lastUpdate > 50) {
|
|
111
|
-
lastUpdate = time;
|
|
112
|
-
if (localAnalyser && localBuffer) {
|
|
113
|
-
setCaptureLevel(getRMS(localAnalyser, localBuffer));
|
|
114
|
-
}
|
|
115
|
-
// Take max RMS across all remote tracks (primary + cohost)
|
|
116
|
-
let maxRemoteLevel = 0;
|
|
117
|
-
for (const [, { analyser, buffer }] of remoteAnalysers) {
|
|
118
|
-
const level = getRMS(analyser, buffer);
|
|
119
|
-
if (level > maxRemoteLevel) maxRemoteLevel = level;
|
|
120
|
-
}
|
|
121
|
-
setPlaybackLevel(maxRemoteLevel);
|
|
122
|
-
}
|
|
123
|
-
rafId = requestAnimationFrame(poll);
|
|
124
|
-
};
|
|
125
|
-
rafId = requestAnimationFrame(poll);
|
|
126
|
-
|
|
127
|
-
// ---------- Cleanup ----------
|
|
128
|
-
const dispose = () => {
|
|
129
|
-
disposed = true;
|
|
130
|
-
cancelAnimationFrame(rafId);
|
|
131
|
-
room.off(RoomEvent.TrackSubscribed, onTrackSubscribed);
|
|
132
|
-
room.off(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);
|
|
133
|
-
remoteAnalysers.clear();
|
|
134
|
-
audioCtx.close().catch(() => {});
|
|
135
|
-
};
|
|
136
|
-
cleanupRef.current = dispose;
|
|
137
|
-
|
|
138
|
-
return dispose;
|
|
139
43
|
}, [room, isConnected]);
|
|
140
44
|
|
|
141
45
|
return { captureLevel, playbackLevel };
|