@alia.onl/sdk 10.1.1 → 11.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +24 -0
- package/package.json +4 -6
- package/src/components/AliaAvatar.tsx +66 -0
- package/src/components/AliaChatContent.tsx +236 -112
- package/src/components/AliaChatHeader.tsx +118 -0
- package/src/components/AliaChatMessageList.tsx +197 -260
- package/src/components/AliaChatScreen.tsx +23 -24
- package/src/components/AliaChatSheet.tsx +70 -316
- package/src/components/AliaWelcomeMessage.tsx +47 -50
- package/src/components/AmbientField.tsx +577 -0
- package/src/components/IdentityMark.tsx +10 -2
- package/src/components/Markdown.tsx +116 -40
- package/src/components/PlanPreviewCard.tsx +71 -55
- package/src/components/QuestionCard.tsx +62 -0
- package/src/components/ThinkingBlock.tsx +47 -0
- package/src/components/VoiceModeIcon.tsx +10 -0
- package/src/components/voice/VoiceControls.tsx +22 -73
- package/src/components/voice/VoiceSession.tsx +2 -1
- package/src/hooks/useAliaChat.ts +22 -15
- package/src/hooks/useAmbientWave.ts +35 -5
- package/src/hooks/useSoundEffects.ts +14 -11
- package/src/hooks/useSpeechToText.ts +9 -2
- package/src/hooks/useTTS.ts +163 -156
- package/src/hooks/useVoiceRoom.ts +228 -170
- package/src/index.ts +26 -6
- package/src/lib/catalogue.ts +17 -4
- package/src/lib/chat-stream.ts +25 -7
- package/src/lib/chat-transport.ts +2 -1
- package/src/lib/citations.ts +63 -0
- package/src/lib/content-utils.ts +3 -2
- package/src/lib/image-data-url.ts +15 -0
- package/src/lib/keyboard.tsx +1 -1
- package/src/lib/labels.tsx +81 -0
- package/src/lib/questions.ts +78 -0
- package/src/lib/speech-recognition.native.ts +15 -4
- package/src/lib/speech-recognition.ts +27 -7
- package/src/lib/task-labels.ts +67 -0
- package/src/lib/tool-call-status.ts +28 -0
- package/src/lib/tool-registry.ts +14 -14
- package/src/lib/voice-text.ts +12 -6
- package/src/lib/voice-turn.ts +4 -1
- package/src/lib/work-log-copy.ts +61 -0
- package/src/lib/work-log.ts +378 -0
- package/src/tools.ts +18 -0
- package/src/types.ts +5 -2
- package/src/voice.ts +12 -4
- package/src/components/Reasoning.tsx +0 -152
- package/src/components/ResearchProgressCard.tsx +0 -159
- package/src/components/ThinkingIndicator.tsx +0 -98
- package/src/components/ui/button.tsx +0 -104
- package/src/components/ui/chat-text-input.tsx +0 -184
- package/src/components/ui/dropdown-menu.tsx +0 -2
- package/src/components/ui/dropdown-menu.web.tsx +0 -383
- package/src/components/ui/prompt-input/actions.tsx +0 -28
- package/src/components/ui/prompt-input/add-menu.tsx +0 -83
- package/src/components/ui/prompt-input/attachments.tsx +0 -181
- package/src/components/ui/prompt-input/autocomplete.tsx +0 -168
- package/src/components/ui/prompt-input/context.tsx +0 -71
- package/src/components/ui/prompt-input/mic-button.tsx +0 -53
- package/src/components/ui/prompt-input/prompt-input.tsx +0 -279
- package/src/components/ui/prompt-input/submit-button.tsx +0 -50
- package/src/components/ui/prompt-input/textarea.tsx +0 -80
- package/src/components/ui/text.tsx +0 -29
- package/src/components/voice/AudioWaveVisualizer.tsx +0 -444
- package/src/components/voice/VoiceOverlay.tsx +0 -75
package/src/hooks/useTTS.ts
CHANGED
|
@@ -146,177 +146,184 @@ export function useTTS(options: UseTTSOptions = {}) {
|
|
|
146
146
|
* gets to rebuild rather than the person getting an error about a button they
|
|
147
147
|
* pressed twice.
|
|
148
148
|
*/
|
|
149
|
-
const playFromUrl = useCallback(
|
|
150
|
-
audioUrl: string,
|
|
151
|
-
|
|
152
|
-
signal: AbortSignal,
|
|
153
|
-
onUnplayable?: () => void,
|
|
154
|
-
) => {
|
|
155
|
-
releasePlayer();
|
|
149
|
+
const playFromUrl = useCallback(
|
|
150
|
+
(audioUrl: string, _messageId: string, signal: AbortSignal, onUnplayable?: () => void) => {
|
|
151
|
+
releasePlayer();
|
|
156
152
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
153
|
+
(async () => {
|
|
154
|
+
try {
|
|
155
|
+
const { createAudioPlayer } = await import('expo-audio');
|
|
156
|
+
if (signal.aborted) return;
|
|
157
|
+
/**
|
|
158
|
+
* `crossOrigin` is web-only and ignored elsewhere, and it is what makes
|
|
159
|
+
* the waveform readable there: a browser will not let an `AnalyserNode`
|
|
160
|
+
* see a cross-origin media element loaded without CORS, so without this
|
|
161
|
+
* `setAudioSamplingEnabled` below silently declines and the field never
|
|
162
|
+
* moves. Clips are served by `GET /media` on the API host, which is a
|
|
163
|
+
* different origin from the app on every deployment.
|
|
164
|
+
*
|
|
165
|
+
* It costs the clip its CORS headers if the app's origin is not on the
|
|
166
|
+
* API's allowlist — but such an origin cannot load a conversation to
|
|
167
|
+
* read aloud in the first place, since `/media` and `/conversations`
|
|
168
|
+
* share that allowlist. Measured against production 2026-08-26:
|
|
169
|
+
* `Origin: https://alia.onl` is answered
|
|
170
|
+
* `access-control-allow-origin: https://alia.onl`, with `Vary: Origin`.
|
|
171
|
+
*/
|
|
172
|
+
const player = createAudioPlayer({ uri: audioUrl }, { crossOrigin: 'anonymous' });
|
|
173
|
+
playerRef.current = player;
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* The real waveform, which is what the ambient field behind the
|
|
177
|
+
* conversation moves to. Enabled BEFORE `play()`: on web this is what
|
|
178
|
+
* builds the analyser, and the sampling loop only starts if the player
|
|
179
|
+
* is already playing or starts afterwards.
|
|
180
|
+
*
|
|
181
|
+
* It can decline, and declining is not an error. Android routes
|
|
182
|
+
* playback sampling through `android.media.audiofx.Visualizer`, which
|
|
183
|
+
* needs `RECORD_AUDIO` already granted — and asking a person for the
|
|
184
|
+
* microphone so that a background can move would be a worse trade than
|
|
185
|
+
* the background not moving. Where it declines no buffer ever arrives,
|
|
186
|
+
* the level stays at zero and the field simply rests. That is the
|
|
187
|
+
* honest reading of "we cannot hear this", and it is deliberately not
|
|
188
|
+
* papered over with motion that does not come from the audio.
|
|
189
|
+
*/
|
|
190
|
+
player.setAudioSamplingEnabled(true);
|
|
191
|
+
const meter = createAudioLevelMeter();
|
|
192
|
+
player.addListener('audioSampleUpdate', (sample: AudioSample) => {
|
|
193
|
+
const frames = sample.channels[0]?.frames;
|
|
194
|
+
if (frames === undefined) return;
|
|
195
|
+
ttsWaveAmplitude.value = meter.push(frames, Date.now());
|
|
196
|
+
});
|
|
178
197
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
* microphone so that a background can move would be a worse trade than
|
|
189
|
-
* the background not moving. Where it declines no buffer ever arrives,
|
|
190
|
-
* the level stays at zero and the field simply rests. That is the
|
|
191
|
-
* honest reading of "we cannot hear this", and it is deliberately not
|
|
192
|
-
* papered over with motion that does not come from the audio.
|
|
193
|
-
*/
|
|
194
|
-
player.setAudioSamplingEnabled(true);
|
|
195
|
-
const meter = createAudioLevelMeter();
|
|
196
|
-
player.addListener('audioSampleUpdate', (sample: AudioSample) => {
|
|
197
|
-
const frames = sample.channels[0]?.frames;
|
|
198
|
-
if (frames === undefined) return;
|
|
199
|
-
ttsWaveAmplitude.value = meter.push(frames, Date.now());
|
|
200
|
-
});
|
|
201
|
-
|
|
202
|
-
player.addListener('playbackStatusUpdate', (status: AudioStatus) => {
|
|
203
|
-
if (signal.aborted || playerRef.current !== player) return;
|
|
204
|
-
if (status.error) {
|
|
205
|
-
releasePlayer();
|
|
206
|
-
if (onUnplayable) {
|
|
207
|
-
onUnplayable();
|
|
198
|
+
player.addListener('playbackStatusUpdate', (status: AudioStatus) => {
|
|
199
|
+
if (signal.aborted || playerRef.current !== player) return;
|
|
200
|
+
if (status.error) {
|
|
201
|
+
releasePlayer();
|
|
202
|
+
if (onUnplayable) {
|
|
203
|
+
onUnplayable();
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
setError(status.error);
|
|
208
207
|
return;
|
|
209
208
|
}
|
|
210
|
-
|
|
209
|
+
if (status.didJustFinish) {
|
|
210
|
+
releasePlayer();
|
|
211
|
+
reset();
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
player.play();
|
|
216
|
+
setPlaybackState('playing');
|
|
217
|
+
} catch {
|
|
218
|
+
if (signal.aborted) return;
|
|
219
|
+
if (onUnplayable) {
|
|
220
|
+
onUnplayable();
|
|
211
221
|
return;
|
|
212
222
|
}
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
223
|
+
setError('Audio playback not available');
|
|
224
|
+
}
|
|
225
|
+
})();
|
|
226
|
+
},
|
|
227
|
+
[releasePlayer, reset, setPlaybackState, setError],
|
|
228
|
+
);
|
|
229
|
+
|
|
230
|
+
const readAloud = useCallback(
|
|
231
|
+
async (messageId: string, text: string, conversationId?: string, audioUrl?: string) => {
|
|
232
|
+
// Read volatile playback state at call time (not via closure) so this
|
|
233
|
+
// callback's identity survives every playback transition. It is passed to
|
|
234
|
+
// memoized message rows — an unstable identity re-renders all of them.
|
|
235
|
+
const { activeMessageId, playbackState } = useTTSStore.getState();
|
|
236
|
+
|
|
237
|
+
// If same message is playing, toggle pause/play
|
|
238
|
+
if (activeMessageId === messageId) {
|
|
239
|
+
if (playbackState === 'playing') {
|
|
240
|
+
playerRef.current?.pause();
|
|
241
|
+
setPlaybackState('paused');
|
|
242
|
+
return;
|
|
243
|
+
}
|
|
244
|
+
if (playbackState === 'paused') {
|
|
245
|
+
playerRef.current?.play();
|
|
246
|
+
setPlaybackState('playing');
|
|
225
247
|
return;
|
|
226
248
|
}
|
|
227
|
-
setError('Audio playback not available');
|
|
228
|
-
}
|
|
229
|
-
})();
|
|
230
|
-
}, [releasePlayer, reset, setPlaybackState, setError]);
|
|
231
|
-
|
|
232
|
-
const readAloud = useCallback(async (
|
|
233
|
-
messageId: string,
|
|
234
|
-
text: string,
|
|
235
|
-
conversationId?: string,
|
|
236
|
-
audioUrl?: string,
|
|
237
|
-
) => {
|
|
238
|
-
// Read volatile playback state at call time (not via closure) so this
|
|
239
|
-
// callback's identity survives every playback transition. It is passed to
|
|
240
|
-
// memoized message rows — an unstable identity re-renders all of them.
|
|
241
|
-
const { activeMessageId, playbackState } = useTTSStore.getState();
|
|
242
|
-
|
|
243
|
-
// If same message is playing, toggle pause/play
|
|
244
|
-
if (activeMessageId === messageId) {
|
|
245
|
-
if (playbackState === 'playing') {
|
|
246
|
-
playerRef.current?.pause();
|
|
247
|
-
setPlaybackState('paused');
|
|
248
|
-
return;
|
|
249
249
|
}
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
250
|
+
|
|
251
|
+
// Stop any current playback
|
|
252
|
+
if (activeMessageId) {
|
|
253
|
+
stop();
|
|
254
254
|
}
|
|
255
|
-
}
|
|
256
255
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
256
|
+
const controller = new AbortController();
|
|
257
|
+
requestRef.current = controller;
|
|
258
|
+
try {
|
|
259
|
+
setActiveMessage(messageId);
|
|
260
|
+
setPlaybackState('loading');
|
|
261
261
|
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
*/
|
|
276
|
-
const synthesize = async () => {
|
|
277
|
-
const token = getToken();
|
|
278
|
-
if (!token) {
|
|
279
|
-
throw new Error('Not authenticated');
|
|
280
|
-
}
|
|
262
|
+
/**
|
|
263
|
+
* Ask the server to make it, and play what comes back.
|
|
264
|
+
*
|
|
265
|
+
* Hoisted out of the cached branch below so a clip that has aged out of
|
|
266
|
+
* storage can fall back to it. `playFromUrl` is called WITHOUT a fallback
|
|
267
|
+
* here: a freshly made clip that will not play is a real failure, and
|
|
268
|
+
* retrying it would be a loop.
|
|
269
|
+
*/
|
|
270
|
+
const synthesize = async () => {
|
|
271
|
+
const token = getToken();
|
|
272
|
+
if (!token) {
|
|
273
|
+
throw new Error('Not authenticated');
|
|
274
|
+
}
|
|
281
275
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
});
|
|
293
|
-
playFromUrl(audioUrl, messageId, controller.signal);
|
|
294
|
-
};
|
|
295
|
-
|
|
296
|
-
/**
|
|
297
|
-
* A stored clip plays straight away, and is remade if it has gone.
|
|
298
|
-
*
|
|
299
|
-
* The row keeps its key long after a `tts/` object ages out of storage,
|
|
300
|
-
* so the link is signed, valid, and points at nothing — a failure only
|
|
301
|
-
* the player ever sees. Remaking it is cheaper than keeping every clip
|
|
302
|
-
* forever, and the person notices nothing.
|
|
303
|
-
*/
|
|
304
|
-
if (audioUrl) {
|
|
305
|
-
playFromUrl(audioUrl, messageId, controller.signal, () => {
|
|
306
|
-
void synthesize().catch((e: unknown) => {
|
|
307
|
-
if (!controller.signal.aborted) setError(errorMessage(e, 'Failed to read aloud'));
|
|
276
|
+
const audioUrl = await requestSpeechClip({
|
|
277
|
+
apiUrl,
|
|
278
|
+
token,
|
|
279
|
+
model: voiceModel,
|
|
280
|
+
input: text,
|
|
281
|
+
voice: getTTSVoice(),
|
|
282
|
+
speed: getTTSSpeed(),
|
|
283
|
+
conversationId,
|
|
284
|
+
messageId,
|
|
285
|
+
signal: controller.signal,
|
|
308
286
|
});
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
}
|
|
287
|
+
playFromUrl(audioUrl, messageId, controller.signal);
|
|
288
|
+
};
|
|
312
289
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
290
|
+
/**
|
|
291
|
+
* A stored clip plays straight away, and is remade if it has gone.
|
|
292
|
+
*
|
|
293
|
+
* The row keeps its key long after a `tts/` object ages out of storage,
|
|
294
|
+
* so the link is signed, valid, and points at nothing — a failure only
|
|
295
|
+
* the player ever sees. Remaking it is cheaper than keeping every clip
|
|
296
|
+
* forever, and the person notices nothing.
|
|
297
|
+
*/
|
|
298
|
+
if (audioUrl) {
|
|
299
|
+
playFromUrl(audioUrl, messageId, controller.signal, () => {
|
|
300
|
+
void synthesize().catch((e: unknown) => {
|
|
301
|
+
if (!controller.signal.aborted) setError(errorMessage(e, 'Failed to read aloud'));
|
|
302
|
+
});
|
|
303
|
+
});
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
await synthesize();
|
|
308
|
+
} catch (e: unknown) {
|
|
309
|
+
if (controller.signal.aborted) return;
|
|
310
|
+
console.error('[TTS] Error:', e);
|
|
311
|
+
setError(errorMessage(e, 'Failed to read aloud'));
|
|
312
|
+
}
|
|
313
|
+
},
|
|
314
|
+
[
|
|
315
|
+
getToken,
|
|
316
|
+
apiUrl,
|
|
317
|
+
voiceModel,
|
|
318
|
+
getTTSVoice,
|
|
319
|
+
getTTSSpeed,
|
|
320
|
+
stop,
|
|
321
|
+
playFromUrl,
|
|
322
|
+
setActiveMessage,
|
|
323
|
+
setPlaybackState,
|
|
324
|
+
setError,
|
|
325
|
+
],
|
|
326
|
+
);
|
|
320
327
|
|
|
321
328
|
// Cleanup on unmount
|
|
322
329
|
useEffect(() => {
|