@alia.onl/sdk 13.1.0 → 15.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +190 -0
- package/README.md +83 -20
- package/package.json +8 -4
- package/src/components/AliaAvatarIcon.tsx +30 -0
- package/src/components/AliaChatContent.tsx +57 -107
- package/src/components/AliaChatMessageList.tsx +2 -10
- package/src/components/AliaChatScreen.tsx +28 -25
- package/src/components/AliaChatSheet.tsx +30 -8
- package/src/components/AliaComposer.tsx +382 -0
- package/src/components/ComposerDropZone.tsx +235 -0
- package/src/components/voice/VoiceControls.tsx +28 -4
- package/src/components/voice/VoiceSession.tsx +23 -7
- package/src/hooks/useAliaActivity.ts +31 -0
- package/src/hooks/useAliaChat.ts +14 -3
- package/src/hooks/useAmbientWave.ts +3 -3
- package/src/hooks/useAttachmentIntake.ts +401 -0
- package/src/hooks/useAudioLevelMonitor.ts +2 -2
- package/src/hooks/useDictation.ts +295 -0
- package/src/hooks/useFocusWhenPresented.ts +41 -0
- package/src/hooks/usePendingQuestions.ts +19 -0
- package/src/hooks/useTTS.ts +24 -63
- package/src/hooks/useVoiceRoom.ts +148 -671
- package/src/index.ts +19 -3
- package/src/lib/activity-socket.ts +102 -0
- package/src/lib/activity-store.ts +53 -0
- package/src/lib/api-url.ts +2 -0
- package/src/lib/attachment-intake.ts +154 -0
- package/src/lib/attachment-tiles.ts +74 -0
- package/src/lib/audio-level.ts +2 -2
- package/src/lib/dictation-recorder-types.ts +37 -0
- package/src/lib/dictation-recorder.native.ts +90 -0
- package/src/lib/dictation-recorder.ts +131 -0
- package/src/lib/dictation.ts +208 -0
- package/src/lib/labels.tsx +50 -1
- package/src/lib/pcm.ts +57 -0
- package/src/lib/speech-language.ts +1 -1
- package/src/lib/speech-messages.ts +43 -65
- package/src/lib/speech-synthesis.ts +104 -9
- package/src/lib/turn-state.ts +15 -0
- package/src/lib/voice-audio-engine-types.ts +59 -0
- package/src/lib/voice-audio-engine.native.ts +232 -0
- package/src/lib/voice-audio-engine.ts +350 -0
- package/src/lib/voice-call.ts +539 -0
- package/src/lib/voice-levels.ts +7 -3
- package/src/lib/voice-protocol.ts +309 -0
- package/src/tools.ts +1 -0
- package/src/types.ts +63 -4
- package/src/voice.ts +31 -10
- package/tsconfig.json +2 -0
- package/src/hooks/useSpeechToText.ts +0 -268
- package/src/lib/speech-recognition-types.ts +0 -108
- package/src/lib/speech-recognition.native.ts +0 -237
- package/src/lib/speech-recognition.ts +0 -308
- package/src/lib/voice-text.ts +0 -198
- package/src/lib/voice-turn.ts +0 -89
|
@@ -8,15 +8,17 @@ import { Loading } from '@oxy.so/bloom/loading';
|
|
|
8
8
|
import { Text } from '@oxy.so/bloom/typography';
|
|
9
9
|
import { VoiceControls as BloomVoiceControls, type VoiceState } from '@oxy.so/bloom/voice';
|
|
10
10
|
import { View } from 'react-native';
|
|
11
|
-
import type { RoomState, AgentState } from '../../types';
|
|
11
|
+
import type { RoomState, AgentState, VoiceCallState } from '../../types';
|
|
12
12
|
|
|
13
13
|
/** Every word the bar shows or announces. An app in another language passes its own. */
|
|
14
14
|
export interface VoiceControlsLabels {
|
|
15
15
|
connecting: string;
|
|
16
|
+
reconnecting: string;
|
|
16
17
|
connected: string;
|
|
17
18
|
listening: string;
|
|
18
19
|
muted: string;
|
|
19
20
|
thinking: string;
|
|
21
|
+
usingTool: string;
|
|
20
22
|
speaking: string;
|
|
21
23
|
mute: string;
|
|
22
24
|
unmute: string;
|
|
@@ -25,10 +27,12 @@ export interface VoiceControlsLabels {
|
|
|
25
27
|
|
|
26
28
|
export const VOICE_CONTROLS_LABELS: VoiceControlsLabels = {
|
|
27
29
|
connecting: 'Connecting...',
|
|
30
|
+
reconnecting: 'Reconnecting...',
|
|
28
31
|
connected: 'Connected',
|
|
29
32
|
listening: 'Listening...',
|
|
30
33
|
muted: 'Muted',
|
|
31
34
|
thinking: 'Thinking...',
|
|
35
|
+
usingTool: 'Working on it...',
|
|
32
36
|
speaking: 'Speaking...',
|
|
33
37
|
mute: 'Mute',
|
|
34
38
|
unmute: 'Unmute',
|
|
@@ -41,6 +45,16 @@ export interface VoiceControlsProps {
|
|
|
41
45
|
isMuted: boolean;
|
|
42
46
|
onToggleMute: () => void;
|
|
43
47
|
onEnd: () => void;
|
|
48
|
+
/**
|
|
49
|
+
* Something the person should know while the call listens — e.g. that the
|
|
50
|
+
* call is about to reach its time limit. Shown in place of "Listening…".
|
|
51
|
+
*/
|
|
52
|
+
notice?: string | null;
|
|
53
|
+
/**
|
|
54
|
+
* The call's full state, when the host has it: a tool running reads as such
|
|
55
|
+
* rather than as "Thinking…".
|
|
56
|
+
*/
|
|
57
|
+
callState?: VoiceCallState;
|
|
44
58
|
/** Overrides for any of {@link VOICE_CONTROLS_LABELS}. */
|
|
45
59
|
labels?: Partial<VoiceControlsLabels>;
|
|
46
60
|
}
|
|
@@ -49,11 +63,15 @@ function getStatusText(
|
|
|
49
63
|
roomState: RoomState,
|
|
50
64
|
agentState: AgentState,
|
|
51
65
|
isMuted: boolean,
|
|
66
|
+
notice: string | null,
|
|
52
67
|
labels: VoiceControlsLabels,
|
|
68
|
+
callState: VoiceCallState | undefined,
|
|
53
69
|
): string {
|
|
54
70
|
if (roomState === 'connecting') return labels.connecting;
|
|
71
|
+
if (roomState === 'reconnecting') return labels.reconnecting;
|
|
55
72
|
if (roomState === 'connected') {
|
|
56
|
-
if (agentState === 'listening') return isMuted ? labels.muted : labels.listening;
|
|
73
|
+
if (agentState === 'listening') return isMuted ? labels.muted : (notice ?? labels.listening);
|
|
74
|
+
if (callState?.kind === 'using-tool') return labels.usingTool;
|
|
57
75
|
if (agentState === 'thinking') return labels.thinking;
|
|
58
76
|
if (agentState === 'speaking') return labels.speaking;
|
|
59
77
|
return labels.connected;
|
|
@@ -74,10 +92,12 @@ export function VoiceControls({
|
|
|
74
92
|
isMuted,
|
|
75
93
|
onToggleMute,
|
|
76
94
|
onEnd,
|
|
95
|
+
notice = null,
|
|
96
|
+
callState,
|
|
77
97
|
labels: labelOverrides,
|
|
78
98
|
}: VoiceControlsProps) {
|
|
79
99
|
const labels = { ...VOICE_CONTROLS_LABELS, ...labelOverrides };
|
|
80
|
-
const statusText = getStatusText(roomState, agentState, isMuted, labels);
|
|
100
|
+
const statusText = getStatusText(roomState, agentState, isMuted, notice, labels, callState);
|
|
81
101
|
|
|
82
102
|
return (
|
|
83
103
|
<View className="items-center gap-4 pb-6 pt-4">
|
|
@@ -102,7 +122,11 @@ export function VoiceControls({
|
|
|
102
122
|
/>
|
|
103
123
|
) : null}
|
|
104
124
|
|
|
105
|
-
{roomState === 'connecting'
|
|
125
|
+
{roomState === 'connecting' || roomState === 'reconnecting' ? (
|
|
126
|
+
<Loading
|
|
127
|
+
accessibilityLabel={roomState === 'connecting' ? labels.connecting : labels.reconnecting}
|
|
128
|
+
/>
|
|
129
|
+
) : null}
|
|
106
130
|
</View>
|
|
107
131
|
);
|
|
108
132
|
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* A live voice call, mounted only while voice mode is on.
|
|
3
3
|
*
|
|
4
4
|
* The chat shell receives it as the `voiceSession` prop rather than importing
|
|
5
|
-
* it, so a text-only consumer never compiles the
|
|
5
|
+
* it, so a text-only consumer never compiles the call, its audio engine or
|
|
6
6
|
* the call UI — which is the reason the `@alia.onl/sdk/voice` entry exists.
|
|
7
7
|
*
|
|
8
8
|
* Mounting starts the call; unmounting ends it (`useVoiceRoom` stops listening
|
|
@@ -17,9 +17,18 @@ import { useAudioLevels } from '../../hooks/useAudioLevels';
|
|
|
17
17
|
import { VoiceControls } from './VoiceControls';
|
|
18
18
|
import type { VoiceSessionProps } from '../../types';
|
|
19
19
|
|
|
20
|
-
export function VoiceSession({
|
|
20
|
+
export function VoiceSession({
|
|
21
|
+
apiUrl,
|
|
22
|
+
conversationId,
|
|
23
|
+
agentId,
|
|
24
|
+
locale,
|
|
25
|
+
voice,
|
|
26
|
+
onStateChange,
|
|
27
|
+
onEnd,
|
|
28
|
+
}: VoiceSessionProps) {
|
|
21
29
|
const {
|
|
22
30
|
room,
|
|
31
|
+
callState,
|
|
23
32
|
roomState,
|
|
24
33
|
agentState,
|
|
25
34
|
isMuted,
|
|
@@ -28,7 +37,13 @@ export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps
|
|
|
28
37
|
isConnected,
|
|
29
38
|
connect,
|
|
30
39
|
toggleMute,
|
|
31
|
-
} = useVoiceRoom({
|
|
40
|
+
} = useVoiceRoom({
|
|
41
|
+
apiUrl,
|
|
42
|
+
...(conversationId === undefined ? {} : { conversationId }),
|
|
43
|
+
...(agentId === undefined ? {} : { agentId }),
|
|
44
|
+
...(locale === undefined ? {} : { lang: locale }),
|
|
45
|
+
...(voice === undefined ? {} : { voicePreference: voice }),
|
|
46
|
+
});
|
|
32
47
|
|
|
33
48
|
const { captureLevel, playbackLevel } = useAudioLevelMonitor(room, isConnected);
|
|
34
49
|
const { waveAmplitude } = useAudioLevels({
|
|
@@ -44,21 +59,21 @@ export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps
|
|
|
44
59
|
useEffect(() => {
|
|
45
60
|
if (dialedRef.current) return;
|
|
46
61
|
dialedRef.current = true;
|
|
47
|
-
connect();
|
|
62
|
+
void connect();
|
|
48
63
|
}, [connect]);
|
|
49
64
|
|
|
50
65
|
useEffect(() => {
|
|
51
66
|
onStateChange({ isConnected, agentState, waveAmplitude, messages });
|
|
52
67
|
}, [isConnected, agentState, waveAmplitude, messages, onStateChange]);
|
|
53
68
|
|
|
54
|
-
// End the call on failure, or once a
|
|
69
|
+
// End the call on failure, or once a call we actually reached ends. The
|
|
55
70
|
// "reached" flag is what keeps the pre-connect `disconnected` state — which
|
|
56
71
|
// every session starts in — from ending the call before it begins.
|
|
57
72
|
const reachedRoomRef = useRef(false);
|
|
58
73
|
useEffect(() => {
|
|
59
|
-
if (roomState
|
|
74
|
+
if (roomState !== 'disconnected') reachedRoomRef.current = true;
|
|
60
75
|
if (error || roomState === 'error') {
|
|
61
|
-
onEnd(error ?? VOICE_ERROR_MESSAGES['
|
|
76
|
+
onEnd(error ?? VOICE_ERROR_MESSAGES['realtime-failed']);
|
|
62
77
|
return;
|
|
63
78
|
}
|
|
64
79
|
if (roomState === 'disconnected' && reachedRoomRef.current) onEnd();
|
|
@@ -71,6 +86,7 @@ export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps
|
|
|
71
86
|
isMuted={isMuted}
|
|
72
87
|
onToggleMute={toggleMute}
|
|
73
88
|
onEnd={onEnd}
|
|
89
|
+
callState={callState}
|
|
74
90
|
/>
|
|
75
91
|
);
|
|
76
92
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { useEffect } from 'react';
|
|
2
|
+
import { useOxy } from '@oxy.so/services/ui/client';
|
|
3
|
+
import type { AliaAvatarState } from '../components/AliaAvatar';
|
|
4
|
+
import { selectAliaActivity, useAliaActivityStore } from '../lib/activity-store';
|
|
5
|
+
import { acquireActivitySocket } from '../lib/activity-socket';
|
|
6
|
+
import { ALIA_API_URL } from '../lib/api-url';
|
|
7
|
+
|
|
8
|
+
export interface UseAliaActivityOptions {
|
|
9
|
+
/** The API the activity socket connects to. Defaults like `useAliaChat`'s `apiUrl`. */
|
|
10
|
+
apiUrl?: string;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* What Alia is doing right now, for any surface that wants to show it — an "Ask Alia" button's
|
|
15
|
+
* avatar, a status dot. It is the busiest of:
|
|
16
|
+
*
|
|
17
|
+
* - a turn running in this app (`useAliaChat`, so the sheet drives every button), and
|
|
18
|
+
* - her runs for the signed-in person elsewhere, reported by the server over one shared socket
|
|
19
|
+
* (background work, another device), connected only while someone is signed in AND using this.
|
|
20
|
+
*/
|
|
21
|
+
export function useAliaActivity(options: UseAliaActivityOptions = {}): AliaAvatarState {
|
|
22
|
+
const { apiUrl = ALIA_API_URL } = options;
|
|
23
|
+
const { oxyServices, isAuthenticated } = useOxy();
|
|
24
|
+
|
|
25
|
+
useEffect(() => {
|
|
26
|
+
if (!isAuthenticated) return;
|
|
27
|
+
return acquireActivitySocket(apiUrl, () => oxyServices.session.accessToken);
|
|
28
|
+
}, [apiUrl, isAuthenticated, oxyServices]);
|
|
29
|
+
|
|
30
|
+
return useAliaActivityStore(selectAliaActivity);
|
|
31
|
+
}
|
package/src/hooks/useAliaChat.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import React, { useCallback, useEffect, useRef, useState } from 'react';
|
|
1
|
+
import React, { useCallback, useEffect, useId, useRef, useState } from 'react';
|
|
2
2
|
import { useOxy } from '@oxy.so/services/ui/client';
|
|
3
3
|
import type { ChatMessage, ToolInvocation } from '../types';
|
|
4
4
|
import { getTextFromContent, type MessageContent } from '../lib/content-utils';
|
|
@@ -7,7 +7,9 @@ import { agentMessageId, delegatedAgent } from '../lib/delegation';
|
|
|
7
7
|
import { streamAliaChat } from '../lib/chat-transport';
|
|
8
8
|
import type { AliaChatStreamEvent } from '../lib/chat-stream';
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
import { ALIA_API_URL } from '../lib/api-url';
|
|
11
|
+
import { useAliaActivityStore } from '../lib/activity-store';
|
|
12
|
+
import { turnActivityState } from '../lib/turn-state';
|
|
11
13
|
|
|
12
14
|
export interface UseAliaChatOptions {
|
|
13
15
|
/**
|
|
@@ -129,7 +131,7 @@ function waitWithAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T>
|
|
|
129
131
|
* handler during its recorded compatibility window.
|
|
130
132
|
*/
|
|
131
133
|
export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn {
|
|
132
|
-
const { apiUrl =
|
|
134
|
+
const { apiUrl = ALIA_API_URL, model, clientContext, accessToken: accessTokenProp } = options;
|
|
133
135
|
const directoryRef = useRef(new Map<string, { id: string; character?: unknown }>());
|
|
134
136
|
directoryRef.current = new Map((options.agents ?? []).map((agent) => [agent.id, agent]));
|
|
135
137
|
const requestUrl = '/v1/chat/completions';
|
|
@@ -147,6 +149,15 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
|
|
|
147
149
|
messagesRef.current = messages;
|
|
148
150
|
}, [messages]);
|
|
149
151
|
|
|
152
|
+
// The turn's lifecycle, published for every "Ask Alia" button (`useAliaActivity`).
|
|
153
|
+
const activitySource = useId();
|
|
154
|
+
const activity = turnActivityState(messages, isStreaming);
|
|
155
|
+
useEffect(() => {
|
|
156
|
+
const { setLocal } = useAliaActivityStore.getState();
|
|
157
|
+
setLocal(activitySource, activity);
|
|
158
|
+
return () => setLocal(activitySource, 'idle');
|
|
159
|
+
}, [activitySource, activity]);
|
|
160
|
+
|
|
150
161
|
const updateAssistant = useCallback(
|
|
151
162
|
(assistantId: string, updater: (message: ChatMessage) => Partial<ChatMessage>) => {
|
|
152
163
|
if (!mountedRef.current) return;
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
Easing,
|
|
11
11
|
type SharedValue,
|
|
12
12
|
} from 'react-native-reanimated';
|
|
13
|
-
import {
|
|
13
|
+
import { useDictationStore } from './useDictation';
|
|
14
14
|
import { LEVEL_ATTACK_MS, LEVEL_RELEASE_MS } from '../lib/audio-level';
|
|
15
15
|
import type { AgentState } from '../types';
|
|
16
16
|
|
|
@@ -95,8 +95,8 @@ export function useAmbientWave({
|
|
|
95
95
|
// travel takes about three times as long. Tuning the constant moves both, by
|
|
96
96
|
// different amounts. They are shared because one number should tune one
|
|
97
97
|
// feel, not because the two are interchangeable.
|
|
98
|
-
const sttIsRecording =
|
|
99
|
-
const sttMetering =
|
|
98
|
+
const sttIsRecording = useDictationStore((s) => s.isRecording);
|
|
99
|
+
const sttMetering = useDictationStore((s) => s.metering);
|
|
100
100
|
const stt = useSharedValue(0);
|
|
101
101
|
useAnimatedReaction(
|
|
102
102
|
() => (sttIsRecording ? Math.max(0.08, sttMetering) : -1),
|
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
import React from 'react';
|
|
2
|
+
import {
|
|
3
|
+
classifyIntake,
|
|
4
|
+
readFraction,
|
|
5
|
+
type IntakeKind,
|
|
6
|
+
type IntakeRefusal,
|
|
7
|
+
} from '../lib/attachment-intake';
|
|
8
|
+
import type { AliaAttachment } from '../types';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The composer's own file intake: the queue between "the user handed us a
|
|
12
|
+
* file" and "the attachment strip has it".
|
|
13
|
+
*
|
|
14
|
+
* ## There is no upload
|
|
15
|
+
*
|
|
16
|
+
* This is the first thing to know, and it decides everything below. Nothing in
|
|
17
|
+
* Alia uploads an attachment. A picked file becomes a `uri` and sits in the
|
|
18
|
+
* store until the message is sent, at which point the host's send
|
|
19
|
+
* inlines the images into the request body as `data:` URLs and drops the
|
|
20
|
+
* documents on the floor. There is no endpoint, no `FormData`, no XHR — so
|
|
21
|
+
* there are no bytes in flight to measure and nothing an abort could stop.
|
|
22
|
+
*
|
|
23
|
+
* What there IS, on web, is a read. A `File` that arrives by paste or by drop
|
|
24
|
+
* is a handle, not bytes, and it has to be read into a `data:` URL before it
|
|
25
|
+
* can be sent — a real transfer of real bytes, of a size the user chose, that
|
|
26
|
+
* takes real time for anything above a few megabytes. `FileReader` reports
|
|
27
|
+
* that read through `progress` events carrying `lengthComputable`, `loaded`
|
|
28
|
+
* and `total`, and it can be stopped with `abort()`. So the progress this
|
|
29
|
+
* queue shows is measured and the cancel button ends something: they are the
|
|
30
|
+
* read, honestly labelled, and not a stand-in for an upload that does not
|
|
31
|
+
* exist. When the browser declines to make the read measurable the tile says
|
|
32
|
+
* so by showing indeterminate activity — see `readFraction`.
|
|
33
|
+
*
|
|
34
|
+
* ## Why the queue is separate from the attachment list
|
|
35
|
+
*
|
|
36
|
+
* A file that is still being read is not an attachment yet: it has no `uri`,
|
|
37
|
+
* so `buildMessageContent` would silently skip it, and the strip would show a
|
|
38
|
+
* tile that contributes nothing to the message. It also cannot be REPAIRED in
|
|
39
|
+
* place — `PromptInput` only forwards an `onUpdateAttachment` when its consumer
|
|
40
|
+
* supplies one, and the chat page supplies `attachments`, `onAddAttachment`
|
|
41
|
+
* and `onRemoveAttachment` and no updater at all, so every `updateAttachment`
|
|
42
|
+
* from inside the composer lands in internal state that the controlled list
|
|
43
|
+
* shadows and nothing ever renders. (That is why the old paste path reached
|
|
44
|
+
* around the composer and called `useStore.getState().updateAttachment`
|
|
45
|
+
* directly.) Holding the in-flight files here instead means the strip's
|
|
46
|
+
* pending rows work the same whether the consumer wired an updater or not, and
|
|
47
|
+
* an attachment is only ever handed over once it has bytes behind it.
|
|
48
|
+
*/
|
|
49
|
+
|
|
50
|
+
/** A file the composer is working on, has failed to read, or will not take. */
|
|
51
|
+
export interface IntakeItem {
|
|
52
|
+
id: string;
|
|
53
|
+
name: string;
|
|
54
|
+
size: number;
|
|
55
|
+
mimeType: string;
|
|
56
|
+
kind: IntakeKind;
|
|
57
|
+
/**
|
|
58
|
+
* `reading` while the bytes move, `failed` once they did not, `refused` for
|
|
59
|
+
* a file that never started — too big, or a dropped folder.
|
|
60
|
+
*
|
|
61
|
+
* A refusal is a queue entry rather than a toast for two reasons. It stays
|
|
62
|
+
* put: a drop of six files where one was too large is a sentence the user
|
|
63
|
+
* needs beside the five that worked, not one that slides away after four
|
|
64
|
+
* seconds. And it keeps the composer free of a notification dependency —
|
|
65
|
+
* `@oxy.so/bloom/toast` is a module the composer's own test suites do not
|
|
66
|
+
* stub, and a component that cannot be mounted in a test is a component
|
|
67
|
+
* whose behaviour nothing pins.
|
|
68
|
+
*/
|
|
69
|
+
status: 'reading' | 'failed' | 'refused';
|
|
70
|
+
/**
|
|
71
|
+
* How far the read has got, 0–1 — or `null` for "no measurable progress",
|
|
72
|
+
* which the tile must render as indeterminate activity rather than as a
|
|
73
|
+
* number. Never a timer, never a guess.
|
|
74
|
+
*/
|
|
75
|
+
fraction: number | null;
|
|
76
|
+
/** Set only on a `refused` row: which sentence to show. */
|
|
77
|
+
refusal?: IntakeRefusal;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** A started read, and the one thing a caller can do to it. */
|
|
81
|
+
export interface ReadHandle {
|
|
82
|
+
abort: () => void;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export interface ReadHandlers {
|
|
86
|
+
onProgress: (fraction: number | null) => void;
|
|
87
|
+
onDone: (dataUrl: string) => void;
|
|
88
|
+
onFailed: () => void;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Begins a read and hands back the way to stop it. Injectable because
|
|
93
|
+
* `FileReader` is a browser API and the tests run in node — and because a test
|
|
94
|
+
* that cannot drive a read step by step cannot pin what cancel and retry do.
|
|
95
|
+
*/
|
|
96
|
+
export type StartRead = (file: Blob, handlers: ReadHandlers) => ReadHandle;
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* The real reader: `FileReader`, the app's ONLY genuine cancellation point in
|
|
100
|
+
* the whole attachment path.
|
|
101
|
+
*/
|
|
102
|
+
const startFileRead: StartRead = (file, handlers) => {
|
|
103
|
+
const reader = new FileReader();
|
|
104
|
+
reader.onprogress = (event) => handlers.onProgress(readFraction(event));
|
|
105
|
+
reader.onload = () => {
|
|
106
|
+
const result = reader.result;
|
|
107
|
+
// A `readAsDataURL` result is always a string when it succeeded; anything
|
|
108
|
+
// else means the read did not produce what the message needs, and a
|
|
109
|
+
// failure the user can retry is the truthful reading of that.
|
|
110
|
+
if (typeof result === 'string') handlers.onDone(result);
|
|
111
|
+
else handlers.onFailed();
|
|
112
|
+
};
|
|
113
|
+
reader.onerror = () => handlers.onFailed();
|
|
114
|
+
reader.readAsDataURL(file);
|
|
115
|
+
return { abort: () => reader.abort() };
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
export interface AttachmentIntake {
|
|
119
|
+
/** In-flight and failed files, in the order they arrived. */
|
|
120
|
+
items: IntakeItem[];
|
|
121
|
+
/** Take a batch of files from a paste or a drop. */
|
|
122
|
+
accept: (files: readonly File[]) => void;
|
|
123
|
+
/** Stop a read and drop the file. */
|
|
124
|
+
cancel: (id: string) => void;
|
|
125
|
+
/** Read a failed file again, from the file itself. */
|
|
126
|
+
retry: (id: string) => void;
|
|
127
|
+
/** Give up on a failed or refused file. */
|
|
128
|
+
dismiss: (id: string) => void;
|
|
129
|
+
/** True while anything is still being read — nothing may be sent yet. */
|
|
130
|
+
isBusy: boolean;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export interface AttachmentIntakeOptions {
|
|
134
|
+
addAttachment: (attachment: AliaAttachment) => void;
|
|
135
|
+
/**
|
|
136
|
+
* What the host's send can carry. Omitted, both; a file of a kind left out is refused
|
|
137
|
+
* (`unsupported`) instead of landing as a tile the send would drop.
|
|
138
|
+
*/
|
|
139
|
+
kinds?: readonly IntakeKind[];
|
|
140
|
+
/** Swapped out by the tests; the real one is `startFileRead`. */
|
|
141
|
+
startRead?: StartRead;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const BOTH_KINDS: readonly IntakeKind[] = ['image', 'document'];
|
|
145
|
+
|
|
146
|
+
/** Ids that cannot collide, without a `Math.random()` anywhere near progress. */
|
|
147
|
+
let sequence = 0;
|
|
148
|
+
const nextId = (prefix: string) => `${prefix}-${Date.now()}-${++sequence}`;
|
|
149
|
+
|
|
150
|
+
export function useAttachmentIntake({
|
|
151
|
+
addAttachment,
|
|
152
|
+
kinds = BOTH_KINDS,
|
|
153
|
+
startRead = startFileRead,
|
|
154
|
+
}: AttachmentIntakeOptions): AttachmentIntake {
|
|
155
|
+
const [items, setItems] = React.useState<IntakeItem[]>([]);
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* The live reads and the files behind them, by item id.
|
|
159
|
+
*
|
|
160
|
+
* Refs rather than state, and deliberately so: a read handle is not
|
|
161
|
+
* something the strip draws, and re-rendering the composer every time a
|
|
162
|
+
* reader is registered would be a render per file per keystroke-free tick.
|
|
163
|
+
* More importantly `cancel` must reach the reader that is running NOW — a
|
|
164
|
+
* handle captured in a closure over state would be one render stale, which
|
|
165
|
+
* for a cancel button means aborting the read before last.
|
|
166
|
+
*
|
|
167
|
+
* The file is kept alongside for exactly one reason: retry has to re-read
|
|
168
|
+
* something, and the `File` is the only thing that still holds the bytes.
|
|
169
|
+
* Both maps are cleared the moment an item leaves the queue, which is what
|
|
170
|
+
* makes "release the reading resources" true rather than aspirational.
|
|
171
|
+
*/
|
|
172
|
+
const reads = React.useRef(new Map<string, ReadHandle>());
|
|
173
|
+
const sources = React.useRef(new Map<string, File>());
|
|
174
|
+
|
|
175
|
+
const forget = React.useCallback((id: string) => {
|
|
176
|
+
reads.current.get(id)?.abort();
|
|
177
|
+
reads.current.delete(id);
|
|
178
|
+
sources.current.delete(id);
|
|
179
|
+
}, []);
|
|
180
|
+
|
|
181
|
+
const begin = React.useCallback(
|
|
182
|
+
(id: string, file: File, kind: IntakeKind) => {
|
|
183
|
+
/**
|
|
184
|
+
* Whether this read is still the one we are waiting on.
|
|
185
|
+
*
|
|
186
|
+
* `abort()` is a request, not a guarantee: a `FileReader` may still
|
|
187
|
+
* deliver a queued event afterwards, and some engines fire `load` for a
|
|
188
|
+
* read that had already completed in the background when the abort
|
|
189
|
+
* arrived. Without this check a cancelled file lands in the attachment
|
|
190
|
+
* list a beat after the user removed it — the row comes BACK — which is
|
|
191
|
+
* the failure mode that makes a cancel button worse than none.
|
|
192
|
+
*
|
|
193
|
+
* The map is the record of what is live, so asking it is also what makes
|
|
194
|
+
* "cancel" and "unmount" mean the same thing here.
|
|
195
|
+
*/
|
|
196
|
+
const live = () => reads.current.has(id);
|
|
197
|
+
|
|
198
|
+
/*
|
|
199
|
+
* Registered BEFORE the read starts, then swapped for the real handle.
|
|
200
|
+
* A cached file can be read synchronously — the browser has the bytes
|
|
201
|
+
* already — and `onDone` would then run inside `startRead`, before there
|
|
202
|
+
* was anything in the map to say this read was live. The placeholder is
|
|
203
|
+
* what stops that completion being mistaken for a stale one and dropped.
|
|
204
|
+
*/
|
|
205
|
+
const placeholder: ReadHandle = { abort: () => {} };
|
|
206
|
+
reads.current.set(id, placeholder);
|
|
207
|
+
|
|
208
|
+
const handle = startRead(file, {
|
|
209
|
+
onProgress: (fraction) => {
|
|
210
|
+
if (!live()) return;
|
|
211
|
+
setItems((prev) =>
|
|
212
|
+
prev.map((item) =>
|
|
213
|
+
item.id === id && item.status === 'reading' ? { ...item, fraction } : item,
|
|
214
|
+
),
|
|
215
|
+
);
|
|
216
|
+
},
|
|
217
|
+
onDone: (dataUrl) => {
|
|
218
|
+
if (!live()) return;
|
|
219
|
+
// The read is over before the attachment exists, so the handle and
|
|
220
|
+
// the file go first: a completed reader kept in the map is a `File`
|
|
221
|
+
// kept alive, which for a 20 MiB drop is 20 MiB held for the life of
|
|
222
|
+
// the composer.
|
|
223
|
+
reads.current.delete(id);
|
|
224
|
+
sources.current.delete(id);
|
|
225
|
+
setItems((prev) => prev.filter((item) => item.id !== id));
|
|
226
|
+
addAttachment({
|
|
227
|
+
id,
|
|
228
|
+
uri: dataUrl,
|
|
229
|
+
type: kind,
|
|
230
|
+
name: file.name,
|
|
231
|
+
size: file.size,
|
|
232
|
+
mimeType: file.type || 'application/octet-stream',
|
|
233
|
+
});
|
|
234
|
+
},
|
|
235
|
+
onFailed: () => {
|
|
236
|
+
if (!live()) return;
|
|
237
|
+
// The reader is spent; the FILE is not, and retry needs it. This is
|
|
238
|
+
// the one path that keeps a source around after its read ended.
|
|
239
|
+
reads.current.delete(id);
|
|
240
|
+
setItems((prev) =>
|
|
241
|
+
prev.map((item) =>
|
|
242
|
+
item.id === id ? { ...item, status: 'failed', fraction: null } : item,
|
|
243
|
+
),
|
|
244
|
+
);
|
|
245
|
+
},
|
|
246
|
+
});
|
|
247
|
+
// Only if the placeholder is still there. If the read already finished
|
|
248
|
+
// or failed inside the call above, the map has been moved on and writing
|
|
249
|
+
// the handle back would leave a spent reader registered as live.
|
|
250
|
+
if (reads.current.get(id) === placeholder) reads.current.set(id, handle);
|
|
251
|
+
},
|
|
252
|
+
[addAttachment, startRead],
|
|
253
|
+
);
|
|
254
|
+
|
|
255
|
+
const accept = React.useCallback(
|
|
256
|
+
(files: readonly File[]) => {
|
|
257
|
+
const queued: IntakeItem[] = [];
|
|
258
|
+
const starts: Array<() => void> = [];
|
|
259
|
+
|
|
260
|
+
for (const file of files) {
|
|
261
|
+
const name = file.name || '';
|
|
262
|
+
const verdict = classifyIntake({
|
|
263
|
+
name,
|
|
264
|
+
mimeType: file.type || '',
|
|
265
|
+
size: file.size,
|
|
266
|
+
});
|
|
267
|
+
const id = nextId('intake');
|
|
268
|
+
const refusal = !verdict.accepted
|
|
269
|
+
? verdict.refusal
|
|
270
|
+
: kinds.includes(verdict.kind)
|
|
271
|
+
? undefined
|
|
272
|
+
: ('unsupported' as const);
|
|
273
|
+
if (refusal !== undefined) {
|
|
274
|
+
// A file that silently fails to attach is the worst of the three
|
|
275
|
+
// outcomes: the user sends the message believing the picture went
|
|
276
|
+
// with it. The row names the file, because a drop is usually a batch
|
|
277
|
+
// and "one of them didn't work" is no answer when four were dropped.
|
|
278
|
+
queued.push({
|
|
279
|
+
id,
|
|
280
|
+
name: name || 'file',
|
|
281
|
+
size: file.size,
|
|
282
|
+
mimeType: file.type || '',
|
|
283
|
+
kind: 'document',
|
|
284
|
+
status: 'refused',
|
|
285
|
+
fraction: null,
|
|
286
|
+
refusal,
|
|
287
|
+
});
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
if (!verdict.accepted) continue;
|
|
291
|
+
if (verdict.kind === 'document') {
|
|
292
|
+
/*
|
|
293
|
+
* A document never gets read. `buildMessageContent` filters the
|
|
294
|
+
* attachment list down to `type === 'image'`, so a document's bytes
|
|
295
|
+
* go nowhere and reading them would be a progress bar over work that
|
|
296
|
+
* exists only to produce a string nobody consumes. The object URL is
|
|
297
|
+
* what the web pickers produce for the same file, it is instant, and
|
|
298
|
+
* the draft store gives it back when the tile is removed.
|
|
299
|
+
*/
|
|
300
|
+
addAttachment({
|
|
301
|
+
id,
|
|
302
|
+
uri:
|
|
303
|
+
typeof URL !== 'undefined' && typeof URL.createObjectURL === 'function'
|
|
304
|
+
? URL.createObjectURL(file)
|
|
305
|
+
: '',
|
|
306
|
+
type: 'document',
|
|
307
|
+
name: name || 'file',
|
|
308
|
+
size: file.size,
|
|
309
|
+
mimeType: file.type || 'application/octet-stream',
|
|
310
|
+
});
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
sources.current.set(id, file);
|
|
315
|
+
queued.push({
|
|
316
|
+
id,
|
|
317
|
+
name: name || 'image',
|
|
318
|
+
size: file.size,
|
|
319
|
+
mimeType: file.type || 'image/*',
|
|
320
|
+
kind: verdict.kind,
|
|
321
|
+
status: 'reading',
|
|
322
|
+
// Not zero. Zero is a claim that nothing has moved yet; `null` is the
|
|
323
|
+
// truth, which is that nothing has been reported yet.
|
|
324
|
+
fraction: null,
|
|
325
|
+
});
|
|
326
|
+
starts.push(() => begin(id, file, verdict.kind));
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
if (queued.length > 0) setItems((prev) => [...prev, ...queued]);
|
|
330
|
+
// Started after the rows exist, because a synchronous reader — the fake
|
|
331
|
+
// one in the tests, and a cached file in some browsers — can complete
|
|
332
|
+
// inside `startRead`, and a completion for a row that has not been added
|
|
333
|
+
// yet would be filtered against a list it is not in and then re-added by
|
|
334
|
+
// the batch above as a ghost that never finishes.
|
|
335
|
+
for (const start of starts) start();
|
|
336
|
+
},
|
|
337
|
+
[addAttachment, begin, kinds],
|
|
338
|
+
);
|
|
339
|
+
|
|
340
|
+
const cancel = React.useCallback(
|
|
341
|
+
(id: string) => {
|
|
342
|
+
forget(id);
|
|
343
|
+
setItems((prev) => prev.filter((item) => item.id !== id));
|
|
344
|
+
},
|
|
345
|
+
[forget],
|
|
346
|
+
);
|
|
347
|
+
|
|
348
|
+
const dismiss = cancel;
|
|
349
|
+
|
|
350
|
+
const retry = React.useCallback(
|
|
351
|
+
(id: string) => {
|
|
352
|
+
// The file IS the guard. A refused row never had one — nothing was
|
|
353
|
+
// read, and reading the same 30 MB again would reach the same verdict —
|
|
354
|
+
// and a cancelled one let go of it, so neither can start a second read
|
|
355
|
+
// by pressing a button the tile does not draw for them anyway.
|
|
356
|
+
const file = sources.current.get(id);
|
|
357
|
+
if (file === undefined) return;
|
|
358
|
+
setItems((prev) =>
|
|
359
|
+
prev.map((item) =>
|
|
360
|
+
item.id === id ? { ...item, status: 'reading', fraction: null } : item,
|
|
361
|
+
),
|
|
362
|
+
);
|
|
363
|
+
// `"image"` is not an assumption: a document is handed over on arrival
|
|
364
|
+
// and never queues, so the only thing that can hold a source is a
|
|
365
|
+
// picture being read.
|
|
366
|
+
begin(id, file, 'image');
|
|
367
|
+
},
|
|
368
|
+
[begin],
|
|
369
|
+
);
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* Nothing survives the composer going away.
|
|
373
|
+
*
|
|
374
|
+
* An in-flight `FileReader` holds its `File`, and a `File` holds the bytes;
|
|
375
|
+
* a composer unmounted mid-read (navigating away, the drawer closing) would
|
|
376
|
+
* otherwise leave both running until the read finished into a `setItems` on
|
|
377
|
+
* a dead tree. Aborting is the whole of the fix and there is nowhere else to
|
|
378
|
+
* put it.
|
|
379
|
+
*/
|
|
380
|
+
React.useEffect(() => {
|
|
381
|
+
const live = reads.current;
|
|
382
|
+
const held = sources.current;
|
|
383
|
+
return () => {
|
|
384
|
+
for (const handle of live.values()) handle.abort();
|
|
385
|
+
live.clear();
|
|
386
|
+
held.clear();
|
|
387
|
+
};
|
|
388
|
+
}, []);
|
|
389
|
+
|
|
390
|
+
return React.useMemo(
|
|
391
|
+
() => ({
|
|
392
|
+
items,
|
|
393
|
+
accept,
|
|
394
|
+
cancel,
|
|
395
|
+
retry,
|
|
396
|
+
dismiss,
|
|
397
|
+
isBusy: items.some((item) => item.status === 'reading'),
|
|
398
|
+
}),
|
|
399
|
+
[items, accept, cancel, retry, dismiss],
|
|
400
|
+
);
|
|
401
|
+
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* A voice call's live levels, as React state at ~20fps.
|
|
3
3
|
*
|
|
4
|
-
* `useVoiceRoom` writes the microphone level (from the
|
|
5
|
-
* playback level (from the
|
|
4
|
+
* `useVoiceRoom` writes the microphone level (from the captured buffers) and the
|
|
5
|
+
* playback level (from the answer being played) into a `VoiceLevelSource`; this
|
|
6
6
|
* reads them on a clock so a wave can follow them without rendering once per
|
|
7
7
|
* audio buffer. It used to put Web Audio analysers on a LiveKit room's tracks;
|
|
8
8
|
* the signature kept its shape — `room` is now the level source `useVoiceRoom`
|