@alia.onl/sdk 5.1.0 → 7.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +93 -0
- package/LICENSE +202 -0
- package/NOTICE +15 -0
- package/package.json +13 -6
- package/src/components/AliaChatContent.tsx +11 -3
- package/src/components/AliaChatScreen.tsx +3 -3
- package/src/components/AliaChatSheet.tsx +17 -5
- package/src/components/AliaWelcomeMessage.tsx +2 -2
- package/src/components/{AliaMark.tsx → IdentityMark.tsx} +87 -19
- package/src/components/Markdown.tsx +31 -6
- package/src/components/ThinkingIndicator.tsx +2 -2
- package/src/components/ui/button.tsx +2 -2
- package/src/components/ui/chat-text-input.tsx +27 -11
- package/src/components/ui/prompt-input/textarea.tsx +19 -3
- package/src/hooks/useAliaChat.ts +309 -352
- package/src/hooks/useAmbientWave.ts +14 -2
- package/src/hooks/useSpeechToText.ts +4 -2
- package/src/hooks/useTTS.ts +162 -65
- package/src/hooks/useVoiceRoom.ts +26 -2
- package/src/index.ts +15 -2
- package/src/lib/audio-level.ts +129 -0
- package/src/lib/catalogue.ts +209 -0
- package/src/lib/chat-stream.ts +560 -0
- package/src/lib/chat-transport.ts +55 -0
- package/src/lib/config.ts +38 -0
- package/src/lib/tool-registry.ts +1 -1
- package/tsconfig.json +1 -1
|
@@ -11,6 +11,7 @@ import {
|
|
|
11
11
|
type SharedValue,
|
|
12
12
|
} from 'react-native-reanimated';
|
|
13
13
|
import { useSTTStore } from './useSpeechToText';
|
|
14
|
+
import { LEVEL_ATTACK_MS, LEVEL_RELEASE_MS } from '../lib/audio-level';
|
|
14
15
|
import type { AgentState } from '../types';
|
|
15
16
|
|
|
16
17
|
export type AmbientWaveMode = 'voice' | 'tts' | 'stt' | 'thinking' | 'idle';
|
|
@@ -27,7 +28,7 @@ export interface UseAmbientWaveOptions {
|
|
|
27
28
|
voice?: AmbientWaveVoice;
|
|
28
29
|
/** Whether TTS ("read aloud") is currently playing. */
|
|
29
30
|
isTTSPlaying: boolean;
|
|
30
|
-
/**
|
|
31
|
+
/** Live "read aloud" playback level, 0..1 (glides to 0 when not playing). */
|
|
31
32
|
ttsWaveAmplitude: SharedValue<number>;
|
|
32
33
|
/** Text-chat generation in flight (thinking or streaming) — gentle wave motion. */
|
|
33
34
|
isGenerating?: boolean;
|
|
@@ -78,6 +79,17 @@ export function useAmbientWave({ voice, isTTSPlaying, ttsWaveAmplitude, isGenera
|
|
|
78
79
|
);
|
|
79
80
|
|
|
80
81
|
// ── STT metering smoothing (floor 0.08, fast attack / slow decay, →0 on stop) ──
|
|
82
|
+
//
|
|
83
|
+
// The recorder publishes on a 100ms poll, so what makes this read as speech
|
|
84
|
+
// rather than as ten steps a second is `withTiming` interpolating between
|
|
85
|
+
// them. Read-aloud reaches the same place by a different mechanism — its
|
|
86
|
+
// player pushes buffers far too fast to route through React — off the same
|
|
87
|
+
// two constants, but NOT with the same curve: this is a fixed-duration ease
|
|
88
|
+
// that is at the target after `LEVEL_ATTACK_MS`, and `audio-level`'s is an
|
|
89
|
+
// exponential whose time constant that is, so 60ms puts it at 63% and full
|
|
90
|
+
// travel takes about three times as long. Tuning the constant moves both, by
|
|
91
|
+
// different amounts. They are shared because one number should tune one
|
|
92
|
+
// feel, not because the two are interchangeable.
|
|
81
93
|
const sttIsRecording = useSTTStore((s) => s.isRecording);
|
|
82
94
|
const sttMetering = useSTTStore((s) => s.metering);
|
|
83
95
|
const stt = useSharedValue(0);
|
|
@@ -88,7 +100,7 @@ export function useAmbientWave({ voice, isTTSPlaying, ttsWaveAmplitude, isGenera
|
|
|
88
100
|
stt.value = withTiming(0, { duration: 300 });
|
|
89
101
|
return;
|
|
90
102
|
}
|
|
91
|
-
const duration = target > stt.value ?
|
|
103
|
+
const duration = target > stt.value ? LEVEL_ATTACK_MS : LEVEL_RELEASE_MS;
|
|
92
104
|
stt.value = withTiming(target, { duration, easing: Easing.bezier(0.33, 1, 0.68, 1) });
|
|
93
105
|
},
|
|
94
106
|
[sttIsRecording, sttMetering],
|
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
} from 'expo-audio';
|
|
10
10
|
import { useOxy } from '@oxyhq/services';
|
|
11
11
|
import { create } from 'zustand';
|
|
12
|
+
import { levelFromDbfs } from '../lib/audio-level';
|
|
12
13
|
|
|
13
14
|
const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
|
|
14
15
|
|
|
@@ -67,8 +68,9 @@ export function useSpeechToText(options: UseSTTOptions = {}) {
|
|
|
67
68
|
if (state === 'recording') {
|
|
68
69
|
let target: number;
|
|
69
70
|
if (recorderState.metering != null) {
|
|
70
|
-
//
|
|
71
|
-
|
|
71
|
+
// The same curve read-aloud playback uses, so one background answers
|
|
72
|
+
// both the same way — see `lib/audio-level.ts`.
|
|
73
|
+
target = levelFromDbfs(recorderState.metering);
|
|
72
74
|
} else {
|
|
73
75
|
// Metering unavailable — simulate gentle activity
|
|
74
76
|
target = 0.12 + Math.random() * 0.18;
|
package/src/hooks/useTTS.ts
CHANGED
|
@@ -1,16 +1,11 @@
|
|
|
1
1
|
import { useCallback, useEffect, useRef } from 'react';
|
|
2
|
-
import type { AudioPlayer, AudioStatus } from 'expo-audio';
|
|
3
|
-
import {
|
|
4
|
-
useSharedValue,
|
|
5
|
-
withRepeat,
|
|
6
|
-
withSequence,
|
|
7
|
-
withTiming,
|
|
8
|
-
cancelAnimation,
|
|
9
|
-
Easing,
|
|
10
|
-
} from 'react-native-reanimated';
|
|
2
|
+
import type { AudioPlayer, AudioSample, AudioStatus } from 'expo-audio';
|
|
3
|
+
import { makeMutable, withTiming } from 'react-native-reanimated';
|
|
11
4
|
import { useOxy } from '@oxyhq/services';
|
|
12
5
|
import { errorMessage } from '../lib/utils';
|
|
13
6
|
import { create } from 'zustand';
|
|
7
|
+
import { createAudioLevelMeter } from '../lib/audio-level';
|
|
8
|
+
import { PREFERRED_VOICE_MODEL_ID } from '../lib/config';
|
|
14
9
|
|
|
15
10
|
const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
|
|
16
11
|
|
|
@@ -21,6 +16,15 @@ export interface UseTTSOptions {
|
|
|
21
16
|
accessToken?: string;
|
|
22
17
|
voice?: 'male' | 'female';
|
|
23
18
|
tone?: 'brief' | 'chill' | 'default';
|
|
19
|
+
/**
|
|
20
|
+
* Speech model. Defaults to the build's `PREFERRED_VOICE_MODEL_ID`.
|
|
21
|
+
*
|
|
22
|
+
* Not checked against `GET /catalogue`: that surface describes what a chat
|
|
23
|
+
* picker may offer and its resolver filters to `chat_visible` entries, so
|
|
24
|
+
* passing a voice identifier through it would substitute a chat model that
|
|
25
|
+
* cannot speak. See `lib/config.ts`.
|
|
26
|
+
*/
|
|
27
|
+
model?: string;
|
|
24
28
|
}
|
|
25
29
|
|
|
26
30
|
// ============== INLINE TTS STORE ==============
|
|
@@ -47,11 +51,45 @@ const useTTSStore = create<TTSStore>((set) => ({
|
|
|
47
51
|
reset: () => set({ activeMessageId: null, playbackState: 'idle', error: null }),
|
|
48
52
|
}));
|
|
49
53
|
|
|
54
|
+
// ============== PLAYBACK LEVEL ==============
|
|
55
|
+
|
|
56
|
+
/** How long the level takes to reach silence once a clip stops sounding. */
|
|
57
|
+
const SILENCE_FADE_MS = 300;
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The live playback level, 0..1, shared by every consumer.
|
|
61
|
+
*
|
|
62
|
+
* At module scope rather than inside the hook because the two ends of it live
|
|
63
|
+
* in different components: the message list calls `readAloud` and therefore
|
|
64
|
+
* owns the player, while the ambient field behind the conversation is rendered
|
|
65
|
+
* somewhere else entirely. Each `useTTS()` call is its own React instance, so a
|
|
66
|
+
* per-instance `useSharedValue` leaves the field reading a value that nothing
|
|
67
|
+
* writes — which is why the animation it replaces had to be driven off the
|
|
68
|
+
* shared playback STATE instead of off the audio. One player, one store, one
|
|
69
|
+
* level.
|
|
70
|
+
*/
|
|
71
|
+
const ttsWaveAmplitude = makeMutable(0);
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Silence is a projection of playback state, not an animation of its own.
|
|
75
|
+
*
|
|
76
|
+
* Every platform stops delivering buffers the instant a clip pauses, ends or
|
|
77
|
+
* fails, so nothing would write the level down again and the field would freeze
|
|
78
|
+
* swollen wherever the last syllable left it. Subscribed once, beside the store
|
|
79
|
+
* that owns the transitions, so a transition added later cannot forget it.
|
|
80
|
+
*/
|
|
81
|
+
useTTSStore.subscribe((state, previous) => {
|
|
82
|
+
if (state.playbackState === 'playing') return;
|
|
83
|
+
if (previous.playbackState !== 'playing') return;
|
|
84
|
+
ttsWaveAmplitude.value = withTiming(0, { duration: SILENCE_FADE_MS });
|
|
85
|
+
});
|
|
86
|
+
|
|
50
87
|
// ============== HOOK ==============
|
|
51
88
|
|
|
52
89
|
export function useTTS(options: UseTTSOptions = {}) {
|
|
53
90
|
const apiUrl = options.apiUrl || API_URL;
|
|
54
91
|
const voicePref = options.voice ?? 'female';
|
|
92
|
+
const voiceModel = options.model ?? PREFERRED_VOICE_MODEL_ID;
|
|
55
93
|
const tonePref = options.tone ?? 'default';
|
|
56
94
|
|
|
57
95
|
const { oxyServices } = useOxy();
|
|
@@ -68,9 +106,6 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
68
106
|
|
|
69
107
|
const playerRef = useRef<AudioPlayer | null>(null);
|
|
70
108
|
|
|
71
|
-
// Simulated wave amplitude for visualization
|
|
72
|
-
const ttsWaveAmplitude = useSharedValue(0);
|
|
73
|
-
|
|
74
109
|
// ============== AUTH ==============
|
|
75
110
|
|
|
76
111
|
const getToken = useCallback((): string | null => {
|
|
@@ -78,24 +113,6 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
78
113
|
return oxyServices.httpService.getAccessToken();
|
|
79
114
|
}, [options.accessToken, oxyServices]);
|
|
80
115
|
|
|
81
|
-
// Animate wave when playing
|
|
82
|
-
useEffect(() => {
|
|
83
|
-
if (playbackState === 'playing') {
|
|
84
|
-
ttsWaveAmplitude.value = withRepeat(
|
|
85
|
-
withSequence(
|
|
86
|
-
withTiming(0.5, { duration: 800, easing: Easing.inOut(Easing.sin) }),
|
|
87
|
-
withTiming(0.2, { duration: 600, easing: Easing.inOut(Easing.sin) }),
|
|
88
|
-
withTiming(0.6, { duration: 700, easing: Easing.inOut(Easing.sin) }),
|
|
89
|
-
withTiming(0.15, { duration: 500, easing: Easing.inOut(Easing.sin) }),
|
|
90
|
-
),
|
|
91
|
-
-1,
|
|
92
|
-
);
|
|
93
|
-
} else {
|
|
94
|
-
cancelAnimation(ttsWaveAmplitude);
|
|
95
|
-
ttsWaveAmplitude.value = withTiming(0, { duration: 300 });
|
|
96
|
-
}
|
|
97
|
-
}, [playbackState]);
|
|
98
|
-
|
|
99
116
|
const getTTSVoice = useCallback(() => {
|
|
100
117
|
return voicePref === 'male' ? 'echo' : 'nova';
|
|
101
118
|
}, [voicePref]);
|
|
@@ -120,18 +137,74 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
120
137
|
reset();
|
|
121
138
|
}, [reset, releasePlayer]);
|
|
122
139
|
|
|
123
|
-
|
|
140
|
+
/**
|
|
141
|
+
* Play a stored clip.
|
|
142
|
+
*
|
|
143
|
+
* `onUnplayable` is how a caller says "I can make this again". Stored audio
|
|
144
|
+
* is not permanent — `tts/` objects age out — and a link to something that is
|
|
145
|
+
* gone fails in the player, not at the request, so the only place to notice
|
|
146
|
+
* is here. Without a fallback the failure is reported; with one, the caller
|
|
147
|
+
* gets to rebuild rather than the person getting an error about a button they
|
|
148
|
+
* pressed twice.
|
|
149
|
+
*/
|
|
150
|
+
const playFromUrl = useCallback((
|
|
151
|
+
audioUrl: string,
|
|
152
|
+
_messageId: string,
|
|
153
|
+
onUnplayable?: () => void,
|
|
154
|
+
) => {
|
|
124
155
|
releasePlayer();
|
|
125
156
|
|
|
126
157
|
(async () => {
|
|
127
158
|
try {
|
|
128
159
|
const { createAudioPlayer } = await import('expo-audio');
|
|
129
|
-
|
|
160
|
+
/**
|
|
161
|
+
* `crossOrigin` is web-only and ignored elsewhere, and it is what makes
|
|
162
|
+
* the waveform readable there: a browser will not let an `AnalyserNode`
|
|
163
|
+
* see a cross-origin media element loaded without CORS, so without this
|
|
164
|
+
* `setAudioSamplingEnabled` below silently declines and the field never
|
|
165
|
+
* moves. Clips are served by `GET /media` on the API host, which is a
|
|
166
|
+
* different origin from the app on every deployment.
|
|
167
|
+
*
|
|
168
|
+
* It costs the clip its CORS headers if the app's origin is not on the
|
|
169
|
+
* API's allowlist — but such an origin cannot load a conversation to
|
|
170
|
+
* read aloud in the first place, since `/media` and `/conversations`
|
|
171
|
+
* share that allowlist. Measured against production 2026-08-26:
|
|
172
|
+
* `Origin: https://alia.onl` is answered
|
|
173
|
+
* `access-control-allow-origin: https://alia.onl`, with `Vary: Origin`.
|
|
174
|
+
*/
|
|
175
|
+
const player = createAudioPlayer({ uri: audioUrl }, { crossOrigin: 'anonymous' });
|
|
130
176
|
playerRef.current = player;
|
|
131
177
|
|
|
178
|
+
/**
|
|
179
|
+
* The real waveform, which is what the ambient field behind the
|
|
180
|
+
* conversation moves to. Enabled BEFORE `play()`: on web this is what
|
|
181
|
+
* builds the analyser, and the sampling loop only starts if the player
|
|
182
|
+
* is already playing or starts afterwards.
|
|
183
|
+
*
|
|
184
|
+
* It can decline, and declining is not an error. Android routes
|
|
185
|
+
* playback sampling through `android.media.audiofx.Visualizer`, which
|
|
186
|
+
* needs `RECORD_AUDIO` already granted — and asking a person for the
|
|
187
|
+
* microphone so that a background can move would be a worse trade than
|
|
188
|
+
* the background not moving. Where it declines no buffer ever arrives,
|
|
189
|
+
* the level stays at zero and the field simply rests. That is the
|
|
190
|
+
* honest reading of "we cannot hear this", and it is deliberately not
|
|
191
|
+
* papered over with motion that does not come from the audio.
|
|
192
|
+
*/
|
|
193
|
+
player.setAudioSamplingEnabled(true);
|
|
194
|
+
const meter = createAudioLevelMeter();
|
|
195
|
+
player.addListener('audioSampleUpdate', (sample: AudioSample) => {
|
|
196
|
+
const frames = sample.channels[0]?.frames;
|
|
197
|
+
if (frames === undefined) return;
|
|
198
|
+
ttsWaveAmplitude.value = meter.push(frames, Date.now());
|
|
199
|
+
});
|
|
200
|
+
|
|
132
201
|
player.addListener('playbackStatusUpdate', (status: AudioStatus) => {
|
|
133
202
|
if (status.error) {
|
|
134
203
|
releasePlayer();
|
|
204
|
+
if (onUnplayable) {
|
|
205
|
+
onUnplayable();
|
|
206
|
+
return;
|
|
207
|
+
}
|
|
135
208
|
setError(status.error);
|
|
136
209
|
return;
|
|
137
210
|
}
|
|
@@ -144,6 +217,10 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
144
217
|
player.play();
|
|
145
218
|
setPlaybackState('playing');
|
|
146
219
|
} catch {
|
|
220
|
+
if (onUnplayable) {
|
|
221
|
+
onUnplayable();
|
|
222
|
+
return;
|
|
223
|
+
}
|
|
147
224
|
setError('Audio playback not available');
|
|
148
225
|
}
|
|
149
226
|
})();
|
|
@@ -183,44 +260,64 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
183
260
|
setActiveMessage(messageId);
|
|
184
261
|
setPlaybackState('loading');
|
|
185
262
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
263
|
+
/**
|
|
264
|
+
* Ask the server to make it, and play what comes back.
|
|
265
|
+
*
|
|
266
|
+
* Hoisted out of the cached branch below so a clip that has aged out of
|
|
267
|
+
* storage can fall back to it. `playFromUrl` is called WITHOUT a fallback
|
|
268
|
+
* here: a freshly made clip that will not play is a real failure, and
|
|
269
|
+
* retrying it would be a loop.
|
|
270
|
+
*/
|
|
271
|
+
const synthesize = async () => {
|
|
272
|
+
const token = getToken();
|
|
273
|
+
if (!token) {
|
|
274
|
+
throw new Error('Not authenticated');
|
|
275
|
+
}
|
|
191
276
|
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
277
|
+
const response = await fetch(`${apiUrl}/v1/audio/speech`, {
|
|
278
|
+
method: 'POST',
|
|
279
|
+
headers: {
|
|
280
|
+
'Content-Type': 'application/json',
|
|
281
|
+
Authorization: `Bearer ${token}`,
|
|
282
|
+
},
|
|
283
|
+
body: JSON.stringify({
|
|
284
|
+
model: voiceModel,
|
|
285
|
+
input: text,
|
|
286
|
+
voice: getTTSVoice(),
|
|
287
|
+
speed: getTTSSpeed(),
|
|
288
|
+
conversationId,
|
|
289
|
+
messageId,
|
|
290
|
+
}),
|
|
291
|
+
});
|
|
197
292
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
body: JSON.stringify({
|
|
205
|
-
model: 'alia-v1-voice',
|
|
206
|
-
input: text,
|
|
207
|
-
voice: getTTSVoice(),
|
|
208
|
-
speed: getTTSSpeed(),
|
|
209
|
-
conversationId,
|
|
210
|
-
messageId,
|
|
211
|
-
}),
|
|
212
|
-
});
|
|
213
|
-
|
|
214
|
-
if (!response.ok) {
|
|
215
|
-
if (response.status === 504) {
|
|
216
|
-
throw new Error('Request timed out — please try again');
|
|
293
|
+
if (!response.ok) {
|
|
294
|
+
if (response.status === 504) {
|
|
295
|
+
throw new Error('Request timed out — please try again');
|
|
296
|
+
}
|
|
297
|
+
const errData = await response.json().catch(() => ({}));
|
|
298
|
+
throw new Error(errData.error?.message || errData.error || 'TTS failed');
|
|
217
299
|
}
|
|
218
|
-
|
|
219
|
-
|
|
300
|
+
|
|
301
|
+
const data = await response.json();
|
|
302
|
+
playFromUrl(data.audioUrl, messageId);
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
/**
|
|
306
|
+
* A stored clip plays straight away, and is remade if it has gone.
|
|
307
|
+
*
|
|
308
|
+
* The row keeps its key long after a `tts/` object ages out of storage,
|
|
309
|
+
* so the link is signed, valid, and points at nothing — a failure only
|
|
310
|
+
* the player ever sees. Remaking it is cheaper than keeping every clip
|
|
311
|
+
* forever, and the person notices nothing.
|
|
312
|
+
*/
|
|
313
|
+
if (audioUrl) {
|
|
314
|
+
playFromUrl(audioUrl, messageId, () => {
|
|
315
|
+
void synthesize().catch((e: unknown) => setError(errorMessage(e, 'Failed to read aloud')));
|
|
316
|
+
});
|
|
317
|
+
return;
|
|
220
318
|
}
|
|
221
319
|
|
|
222
|
-
|
|
223
|
-
playFromUrl(data.audioUrl, messageId);
|
|
320
|
+
await synthesize();
|
|
224
321
|
} catch (e: unknown) {
|
|
225
322
|
console.error('[TTS] Error:', e);
|
|
226
323
|
setError(errorMessage(e, 'Failed to read aloud'));
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
import { useOxy } from '@oxyhq/services';
|
|
21
21
|
import { errorMessage } from '../lib/utils';
|
|
22
22
|
import type { RoomState, AgentState, VoiceMessage, VoiceToolInvocation } from '../types';
|
|
23
|
+
import { PREFERRED_VOICE_MODEL_ID } from '../lib/config';
|
|
23
24
|
|
|
24
25
|
const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
|
|
25
26
|
|
|
@@ -33,6 +34,23 @@ export interface UseVoiceRoomOptions {
|
|
|
33
34
|
apiUrl?: string;
|
|
34
35
|
voicePreference?: 'male' | 'female';
|
|
35
36
|
accessToken?: string;
|
|
37
|
+
/**
|
|
38
|
+
* Voice session model. Defaults to the build's `PREFERRED_VOICE_MODEL_ID`.
|
|
39
|
+
*
|
|
40
|
+
* Not checked against `GET /catalogue`, for the reason given in
|
|
41
|
+
* `lib/config.ts`: the chat resolver filters to `chat_visible` entries and
|
|
42
|
+
* would substitute something that cannot hold a voice session.
|
|
43
|
+
*/
|
|
44
|
+
model?: string;
|
|
45
|
+
/**
|
|
46
|
+
* The agent this session belongs to, when it is a thread with one.
|
|
47
|
+
*
|
|
48
|
+
* Without it the session is ordinary Alia: the API composes the generic
|
|
49
|
+
* prompt and the identity guard says the model's name, so an agent's voice
|
|
50
|
+
* answers as Alia. It is the caller's to supply because only the screen knows
|
|
51
|
+
* whose thread is open.
|
|
52
|
+
*/
|
|
53
|
+
agentId?: string;
|
|
36
54
|
}
|
|
37
55
|
|
|
38
56
|
// ============== TITLE TAG UTILITIES ==============
|
|
@@ -105,6 +123,8 @@ function micDeviceErrorMessage(name: string): string | null {
|
|
|
105
123
|
export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
106
124
|
const apiUrl = options.apiUrl || API_URL;
|
|
107
125
|
const voicePref = options.voicePreference ?? 'female';
|
|
126
|
+
const voiceModel = options.model ?? PREFERRED_VOICE_MODEL_ID;
|
|
127
|
+
const agentId = options.agentId;
|
|
108
128
|
|
|
109
129
|
const [roomState, setRoomState] = useState<RoomState>('disconnected');
|
|
110
130
|
const [agentState, setAgentState] = useState<AgentState>('idle');
|
|
@@ -329,8 +349,12 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
329
349
|
Authorization: `Bearer ${token}`,
|
|
330
350
|
},
|
|
331
351
|
body: JSON.stringify({
|
|
332
|
-
model:
|
|
352
|
+
model: voiceModel,
|
|
333
353
|
voice: voicePref === 'male' ? 'echo' : 'nova',
|
|
354
|
+
// Omitted rather than sent as undefined: the API branches on the key
|
|
355
|
+
// being a string, and a session with no agent must look exactly as it
|
|
356
|
+
// did before this existed.
|
|
357
|
+
...(agentId ? { agentId } : {}),
|
|
334
358
|
}),
|
|
335
359
|
});
|
|
336
360
|
|
|
@@ -408,7 +432,7 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
408
432
|
setRoomState('error');
|
|
409
433
|
cleanup();
|
|
410
434
|
}
|
|
411
|
-
}, [getToken, cleanup, handleDataMessage, apiUrl, voicePref]);
|
|
435
|
+
}, [getToken, cleanup, handleDataMessage, apiUrl, voicePref, voiceModel, agentId]);
|
|
412
436
|
|
|
413
437
|
// ============== DISCONNECT ==============
|
|
414
438
|
|
package/src/index.ts
CHANGED
|
@@ -9,8 +9,8 @@ export { AliaChatScreen } from './components/AliaChatScreen';
|
|
|
9
9
|
export type { AliaChatScreenProps } from './components/AliaChatScreen';
|
|
10
10
|
|
|
11
11
|
// ── Chat UI components ──
|
|
12
|
-
export {
|
|
13
|
-
export type {
|
|
12
|
+
export { IdentityMark } from './components/IdentityMark';
|
|
13
|
+
export type { IdentityMarkProps, IdentityMarkState } from './components/IdentityMark';
|
|
14
14
|
export { ThinkingIndicator } from './components/ThinkingIndicator';
|
|
15
15
|
export { AliaMarkdown } from './components/Markdown';
|
|
16
16
|
export type { AliaMarkdownProps } from './components/Markdown';
|
|
@@ -29,6 +29,19 @@ export type { UseTTSOptions } from './hooks/useTTS';
|
|
|
29
29
|
export { useSpeechToText, useSTTStore } from './hooks/useSpeechToText';
|
|
30
30
|
export type { UseSTTOptions } from './hooks/useSpeechToText';
|
|
31
31
|
|
|
32
|
+
// ── Model catalogue (GET /catalogue) ──
|
|
33
|
+
// Exported so a consumer building its own picker reads the surface the SDK
|
|
34
|
+
// reads, rather than hardcoding an identifier the way this package used to.
|
|
35
|
+
export {
|
|
36
|
+
fetchCatalogue,
|
|
37
|
+
parseCatalogue,
|
|
38
|
+
resolveSelection,
|
|
39
|
+
resolveModelId,
|
|
40
|
+
clearCatalogueCache,
|
|
41
|
+
} from './lib/catalogue';
|
|
42
|
+
export type { CatalogueEntry, CatalogueEntryKind, ModelSelection } from './lib/catalogue';
|
|
43
|
+
export { PREFERRED_CHAT_MODEL_ID, PREFERRED_VOICE_MODEL_ID } from './lib/config';
|
|
44
|
+
|
|
32
45
|
// ── Types ──
|
|
33
46
|
export type {
|
|
34
47
|
ChatMessage,
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How audio becomes the ambient field's 0..1 level — one curve, one pair of
|
|
3
|
+
* time constants, for every source that drives it.
|
|
4
|
+
*
|
|
5
|
+
* Dictation got here first and is the reference: the recorder reports dBFS,
|
|
6
|
+
* `useSpeechToText` normalises it against a -60 floor, and `useAmbientWave`
|
|
7
|
+
* smooths it with a fast attack and a slow decay before it reaches
|
|
8
|
+
* `AmbientField`. That is the behaviour people like, so it is the behaviour
|
|
9
|
+
* everything else adopts rather than re-invents — including "read aloud", which
|
|
10
|
+
* until now drove the same field from a canned `withRepeat` loop that had never
|
|
11
|
+
* heard the clip.
|
|
12
|
+
*
|
|
13
|
+
* The two sources differ only in what they can measure and how fast:
|
|
14
|
+
*
|
|
15
|
+
* - A **recorder** publishes dBFS on a 100ms poll, through React state, and
|
|
16
|
+
* `levelFromDbfs` is the whole conversion.
|
|
17
|
+
* - A **player** publishes raw PCM as fast as the platform samples, so it needs
|
|
18
|
+
* `levelFromFrames` to reach the same dBFS first, and it cannot go through
|
|
19
|
+
* React at all — a render per audio buffer is not affordable. That is why the
|
|
20
|
+
* smoothing here is a time integration it can run per buffer rather than the
|
|
21
|
+
* `withTiming` the 10Hz path uses. The constants are the same; the CURVES are
|
|
22
|
+
* not. `withTiming` is at its target after `LEVEL_ATTACK_MS`, and this is an
|
|
23
|
+
* exponential whose time constant that is — 63% of the way at 60ms, ~95% at
|
|
24
|
+
* 180. Sharing the number keeps one knob for one feel; it does not make the
|
|
25
|
+
* two settle alike, and anyone tuning it should expect the two paths to move
|
|
26
|
+
* by different amounts.
|
|
27
|
+
*
|
|
28
|
+
* This module deliberately imports nothing. It is the part of the metering that
|
|
29
|
+
* can be driven with a buffer and asserted on, and `react-native-reanimated`
|
|
30
|
+
* does not load outside a bundler — a level that could only be observed by
|
|
31
|
+
* playing a real clip could not have a positive control.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* The quietest the field can hear, in dBFS.
|
|
36
|
+
*
|
|
37
|
+
* Silence is -160 on a real meter and negative infinity on a computed one, so a
|
|
38
|
+
* practical floor is what makes the scale usable. -60 is dictation's, and the
|
|
39
|
+
* compression it produces is most of why that field reads well: it answers
|
|
40
|
+
* quiet speech nearly as much as loud, which is what a person watching a
|
|
41
|
+
* background wants, and unlike a linear reading it does not sit near zero for
|
|
42
|
+
* everything below a shout.
|
|
43
|
+
*/
|
|
44
|
+
export const LEVEL_FLOOR_DBFS = -60;
|
|
45
|
+
|
|
46
|
+
/** Rise fast enough to catch the front of a syllable… */
|
|
47
|
+
export const LEVEL_ATTACK_MS = 60;
|
|
48
|
+
/** …and fall slowly enough that the field does not strobe between them. */
|
|
49
|
+
export const LEVEL_RELEASE_MS = 200;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* dBFS, as a recorder reports it, to the field's 0..1.
|
|
53
|
+
*
|
|
54
|
+
* `Number.NEGATIVE_INFINITY` is a legitimate input — it is what digital silence
|
|
55
|
+
* computes to — and clamps to 0 like anything below the floor.
|
|
56
|
+
*/
|
|
57
|
+
export function levelFromDbfs(dbfs: number): number {
|
|
58
|
+
const level = (dbfs - LEVEL_FLOOR_DBFS) / -LEVEL_FLOOR_DBFS;
|
|
59
|
+
return Math.min(1, Math.max(0, level));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* One buffer of PCM frames, as a player reports them, to the field's 0..1.
|
|
64
|
+
*
|
|
65
|
+
* The frames are -1..1 on every platform `expo-audio` supports, so their RMS is
|
|
66
|
+
* an amplitude ratio and `20·log₁₀` is the dBFS a recorder would have reported
|
|
67
|
+
* for the same sound. Both sources end up on one scale.
|
|
68
|
+
*/
|
|
69
|
+
export function levelFromFrames(frames: readonly number[]): number {
|
|
70
|
+
if (frames.length === 0) return 0;
|
|
71
|
+
|
|
72
|
+
let sumSquares = 0;
|
|
73
|
+
for (let index = 0; index < frames.length; index += 1) {
|
|
74
|
+
sumSquares += frames[index] * frames[index];
|
|
75
|
+
}
|
|
76
|
+
const rms = Math.sqrt(sumSquares / frames.length);
|
|
77
|
+
|
|
78
|
+
// log₁₀(0) is -Infinity, which `levelFromDbfs` clamps — but only because it
|
|
79
|
+
// is asked to. Digital silence is exactly this case, so it is the one that
|
|
80
|
+
// matters most.
|
|
81
|
+
return levelFromDbfs(20 * Math.log10(rms));
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export interface AudioLevelMeter {
|
|
85
|
+
/**
|
|
86
|
+
* Fold one buffer into the running level and return it.
|
|
87
|
+
*
|
|
88
|
+
* @param frames PCM frames, each -1..1, from `AudioSample.channels[n].frames`.
|
|
89
|
+
* @param nowMs A wall clock in milliseconds. The buffer rate is not fixed —
|
|
90
|
+
* web samples once per animation frame, the native taps run at the audio
|
|
91
|
+
* hardware's rate — so the smoothing is integrated against real elapsed
|
|
92
|
+
* time. The same sentence has to read the same at 30Hz and at 90Hz.
|
|
93
|
+
*/
|
|
94
|
+
push(frames: readonly number[], nowMs: number): number;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* A meter per clip: the level starts from silence and carries no history from
|
|
99
|
+
* whatever was playing before.
|
|
100
|
+
*/
|
|
101
|
+
export function createAudioLevelMeter(): AudioLevelMeter {
|
|
102
|
+
let level = 0;
|
|
103
|
+
let lastMs: number | null = null;
|
|
104
|
+
|
|
105
|
+
return {
|
|
106
|
+
push(frames, nowMs) {
|
|
107
|
+
// A platform that cannot sample sends nothing rather than zeroes, but an
|
|
108
|
+
// empty buffer is the same statement: no new evidence, so no new level.
|
|
109
|
+
if (frames.length === 0) return level;
|
|
110
|
+
|
|
111
|
+
const target = levelFromFrames(frames);
|
|
112
|
+
|
|
113
|
+
if (lastMs === null) {
|
|
114
|
+
// Nothing to smooth against on the first buffer of a clip. Smoothing it
|
|
115
|
+
// from an invented zero would put a fade-in at the start of every clip
|
|
116
|
+
// that the audio does not have.
|
|
117
|
+
lastMs = nowMs;
|
|
118
|
+
level = target;
|
|
119
|
+
return level;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const elapsed = Math.max(0, nowMs - lastMs);
|
|
123
|
+
lastMs = nowMs;
|
|
124
|
+
const tau = target > level ? LEVEL_ATTACK_MS : LEVEL_RELEASE_MS;
|
|
125
|
+
level += (target - level) * (1 - Math.exp(-elapsed / tau));
|
|
126
|
+
return level;
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
}
|