@alia.onl/sdk 13.1.0 → 15.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/CHANGELOG.md +190 -0
  2. package/README.md +83 -20
  3. package/package.json +8 -4
  4. package/src/components/AliaAvatarIcon.tsx +30 -0
  5. package/src/components/AliaChatContent.tsx +57 -107
  6. package/src/components/AliaChatMessageList.tsx +2 -10
  7. package/src/components/AliaChatScreen.tsx +28 -25
  8. package/src/components/AliaChatSheet.tsx +30 -8
  9. package/src/components/AliaComposer.tsx +382 -0
  10. package/src/components/ComposerDropZone.tsx +235 -0
  11. package/src/components/voice/VoiceControls.tsx +28 -4
  12. package/src/components/voice/VoiceSession.tsx +23 -7
  13. package/src/hooks/useAliaActivity.ts +31 -0
  14. package/src/hooks/useAliaChat.ts +14 -3
  15. package/src/hooks/useAmbientWave.ts +3 -3
  16. package/src/hooks/useAttachmentIntake.ts +401 -0
  17. package/src/hooks/useAudioLevelMonitor.ts +2 -2
  18. package/src/hooks/useDictation.ts +295 -0
  19. package/src/hooks/useFocusWhenPresented.ts +41 -0
  20. package/src/hooks/usePendingQuestions.ts +19 -0
  21. package/src/hooks/useTTS.ts +24 -63
  22. package/src/hooks/useVoiceRoom.ts +148 -671
  23. package/src/index.ts +19 -3
  24. package/src/lib/activity-socket.ts +102 -0
  25. package/src/lib/activity-store.ts +53 -0
  26. package/src/lib/api-url.ts +2 -0
  27. package/src/lib/attachment-intake.ts +154 -0
  28. package/src/lib/attachment-tiles.ts +74 -0
  29. package/src/lib/audio-level.ts +2 -2
  30. package/src/lib/dictation-recorder-types.ts +37 -0
  31. package/src/lib/dictation-recorder.native.ts +90 -0
  32. package/src/lib/dictation-recorder.ts +131 -0
  33. package/src/lib/dictation.ts +208 -0
  34. package/src/lib/labels.tsx +50 -1
  35. package/src/lib/pcm.ts +57 -0
  36. package/src/lib/speech-language.ts +1 -1
  37. package/src/lib/speech-messages.ts +43 -65
  38. package/src/lib/speech-synthesis.ts +104 -9
  39. package/src/lib/turn-state.ts +15 -0
  40. package/src/lib/voice-audio-engine-types.ts +59 -0
  41. package/src/lib/voice-audio-engine.native.ts +232 -0
  42. package/src/lib/voice-audio-engine.ts +350 -0
  43. package/src/lib/voice-call.ts +539 -0
  44. package/src/lib/voice-levels.ts +7 -3
  45. package/src/lib/voice-protocol.ts +309 -0
  46. package/src/tools.ts +1 -0
  47. package/src/types.ts +63 -4
  48. package/src/voice.ts +31 -10
  49. package/tsconfig.json +2 -0
  50. package/src/hooks/useSpeechToText.ts +0 -268
  51. package/src/lib/speech-recognition-types.ts +0 -108
  52. package/src/lib/speech-recognition.native.ts +0 -237
  53. package/src/lib/speech-recognition.ts +0 -308
  54. package/src/lib/voice-text.ts +0 -198
  55. package/src/lib/voice-turn.ts +0 -89
@@ -8,15 +8,17 @@ import { Loading } from '@oxy.so/bloom/loading';
8
8
  import { Text } from '@oxy.so/bloom/typography';
9
9
  import { VoiceControls as BloomVoiceControls, type VoiceState } from '@oxy.so/bloom/voice';
10
10
  import { View } from 'react-native';
11
- import type { RoomState, AgentState } from '../../types';
11
+ import type { RoomState, AgentState, VoiceCallState } from '../../types';
12
12
 
13
13
  /** Every word the bar shows or announces. An app in another language passes its own. */
14
14
  export interface VoiceControlsLabels {
15
15
  connecting: string;
16
+ reconnecting: string;
16
17
  connected: string;
17
18
  listening: string;
18
19
  muted: string;
19
20
  thinking: string;
21
+ usingTool: string;
20
22
  speaking: string;
21
23
  mute: string;
22
24
  unmute: string;
@@ -25,10 +27,12 @@ export interface VoiceControlsLabels {
25
27
 
26
28
  export const VOICE_CONTROLS_LABELS: VoiceControlsLabels = {
27
29
  connecting: 'Connecting...',
30
+ reconnecting: 'Reconnecting...',
28
31
  connected: 'Connected',
29
32
  listening: 'Listening...',
30
33
  muted: 'Muted',
31
34
  thinking: 'Thinking...',
35
+ usingTool: 'Working on it...',
32
36
  speaking: 'Speaking...',
33
37
  mute: 'Mute',
34
38
  unmute: 'Unmute',
@@ -41,6 +45,16 @@ export interface VoiceControlsProps {
41
45
  isMuted: boolean;
42
46
  onToggleMute: () => void;
43
47
  onEnd: () => void;
48
+ /**
49
+ * Something the person should know while the call listens — e.g. that the
50
+ * call is about to reach its time limit. Shown in place of "Listening…".
51
+ */
52
+ notice?: string | null;
53
+ /**
54
+ * The call's full state, when the host has it: a tool running reads as such
55
+ * rather than as "Thinking…".
56
+ */
57
+ callState?: VoiceCallState;
44
58
  /** Overrides for any of {@link VOICE_CONTROLS_LABELS}. */
45
59
  labels?: Partial<VoiceControlsLabels>;
46
60
  }
@@ -49,11 +63,15 @@ function getStatusText(
49
63
  roomState: RoomState,
50
64
  agentState: AgentState,
51
65
  isMuted: boolean,
66
+ notice: string | null,
52
67
  labels: VoiceControlsLabels,
68
+ callState: VoiceCallState | undefined,
53
69
  ): string {
54
70
  if (roomState === 'connecting') return labels.connecting;
71
+ if (roomState === 'reconnecting') return labels.reconnecting;
55
72
  if (roomState === 'connected') {
56
- if (agentState === 'listening') return isMuted ? labels.muted : labels.listening;
73
+ if (agentState === 'listening') return isMuted ? labels.muted : (notice ?? labels.listening);
74
+ if (callState?.kind === 'using-tool') return labels.usingTool;
57
75
  if (agentState === 'thinking') return labels.thinking;
58
76
  if (agentState === 'speaking') return labels.speaking;
59
77
  return labels.connected;
@@ -74,10 +92,12 @@ export function VoiceControls({
74
92
  isMuted,
75
93
  onToggleMute,
76
94
  onEnd,
95
+ notice = null,
96
+ callState,
77
97
  labels: labelOverrides,
78
98
  }: VoiceControlsProps) {
79
99
  const labels = { ...VOICE_CONTROLS_LABELS, ...labelOverrides };
80
- const statusText = getStatusText(roomState, agentState, isMuted, labels);
100
+ const statusText = getStatusText(roomState, agentState, isMuted, notice, labels, callState);
81
101
 
82
102
  return (
83
103
  <View className="items-center gap-4 pb-6 pt-4">
@@ -102,7 +122,11 @@ export function VoiceControls({
102
122
  />
103
123
  ) : null}
104
124
 
105
- {roomState === 'connecting' ? <Loading accessibilityLabel={labels.connecting} /> : null}
125
+ {roomState === 'connecting' || roomState === 'reconnecting' ? (
126
+ <Loading
127
+ accessibilityLabel={roomState === 'connecting' ? labels.connecting : labels.reconnecting}
128
+ />
129
+ ) : null}
106
130
  </View>
107
131
  );
108
132
  }
@@ -2,7 +2,7 @@
2
2
  * A live voice call, mounted only while voice mode is on.
3
3
  *
4
4
  * The chat shell receives it as the `voiceSession` prop rather than importing
5
- * it, so a text-only consumer never compiles the voice loop, speech playback or
5
+ * it, so a text-only consumer never compiles the call, its audio engine or
6
6
  * the call UI — which is the reason the `@alia.onl/sdk/voice` entry exists.
7
7
  *
8
8
  * Mounting starts the call; unmounting ends it (`useVoiceRoom` stops listening
@@ -17,9 +17,18 @@ import { useAudioLevels } from '../../hooks/useAudioLevels';
17
17
  import { VoiceControls } from './VoiceControls';
18
18
  import type { VoiceSessionProps } from '../../types';
19
19
 
20
- export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps) {
20
+ export function VoiceSession({
21
+ apiUrl,
22
+ conversationId,
23
+ agentId,
24
+ locale,
25
+ voice,
26
+ onStateChange,
27
+ onEnd,
28
+ }: VoiceSessionProps) {
21
29
  const {
22
30
  room,
31
+ callState,
23
32
  roomState,
24
33
  agentState,
25
34
  isMuted,
@@ -28,7 +37,13 @@ export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps
28
37
  isConnected,
29
38
  connect,
30
39
  toggleMute,
31
- } = useVoiceRoom({ apiUrl });
40
+ } = useVoiceRoom({
41
+ apiUrl,
42
+ ...(conversationId === undefined ? {} : { conversationId }),
43
+ ...(agentId === undefined ? {} : { agentId }),
44
+ ...(locale === undefined ? {} : { lang: locale }),
45
+ ...(voice === undefined ? {} : { voicePreference: voice }),
46
+ });
32
47
 
33
48
  const { captureLevel, playbackLevel } = useAudioLevelMonitor(room, isConnected);
34
49
  const { waveAmplitude } = useAudioLevels({
@@ -44,21 +59,21 @@ export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps
44
59
  useEffect(() => {
45
60
  if (dialedRef.current) return;
46
61
  dialedRef.current = true;
47
- connect();
62
+ void connect();
48
63
  }, [connect]);
49
64
 
50
65
  useEffect(() => {
51
66
  onStateChange({ isConnected, agentState, waveAmplitude, messages });
52
67
  }, [isConnected, agentState, waveAmplitude, messages, onStateChange]);
53
68
 
54
- // End the call on failure, or once a room we actually reached goes away. The
69
+ // End the call on failure, or once a call we actually reached ends. The
55
70
  // "reached" flag is what keeps the pre-connect `disconnected` state — which
56
71
  // every session starts in — from ending the call before it begins.
57
72
  const reachedRoomRef = useRef(false);
58
73
  useEffect(() => {
59
- if (roomState === 'connected') reachedRoomRef.current = true;
74
+ if (roomState !== 'disconnected') reachedRoomRef.current = true;
60
75
  if (error || roomState === 'error') {
61
- onEnd(error ?? VOICE_ERROR_MESSAGES['turn-failed']);
76
+ onEnd(error ?? VOICE_ERROR_MESSAGES['realtime-failed']);
62
77
  return;
63
78
  }
64
79
  if (roomState === 'disconnected' && reachedRoomRef.current) onEnd();
@@ -71,6 +86,7 @@ export function VoiceSession({ apiUrl, onStateChange, onEnd }: VoiceSessionProps
71
86
  isMuted={isMuted}
72
87
  onToggleMute={toggleMute}
73
88
  onEnd={onEnd}
89
+ callState={callState}
74
90
  />
75
91
  );
76
92
  }
@@ -0,0 +1,31 @@
1
+ import { useEffect } from 'react';
2
+ import { useOxy } from '@oxy.so/services/ui/client';
3
+ import type { AliaAvatarState } from '../components/AliaAvatar';
4
+ import { selectAliaActivity, useAliaActivityStore } from '../lib/activity-store';
5
+ import { acquireActivitySocket } from '../lib/activity-socket';
6
+ import { ALIA_API_URL } from '../lib/api-url';
7
+
8
+ export interface UseAliaActivityOptions {
9
+ /** The API the activity socket connects to. Defaults like `useAliaChat`'s `apiUrl`. */
10
+ apiUrl?: string;
11
+ }
12
+
13
+ /**
14
+ * What Alia is doing right now, for any surface that wants to show it — an "Ask Alia" button's
15
+ * avatar, a status dot. It is the busiest of:
16
+ *
17
+ * - a turn running in this app (`useAliaChat`, so the sheet drives every button), and
18
+ * - her runs for the signed-in person elsewhere, reported by the server over one shared socket
19
+ * (background work, another device), connected only while someone is signed in AND using this.
20
+ */
21
+ export function useAliaActivity(options: UseAliaActivityOptions = {}): AliaAvatarState {
22
+ const { apiUrl = ALIA_API_URL } = options;
23
+ const { oxyServices, isAuthenticated } = useOxy();
24
+
25
+ useEffect(() => {
26
+ if (!isAuthenticated) return;
27
+ return acquireActivitySocket(apiUrl, () => oxyServices.session.accessToken);
28
+ }, [apiUrl, isAuthenticated, oxyServices]);
29
+
30
+ return useAliaActivityStore(selectAliaActivity);
31
+ }
@@ -1,4 +1,4 @@
1
- import React, { useCallback, useEffect, useRef, useState } from 'react';
1
+ import React, { useCallback, useEffect, useId, useRef, useState } from 'react';
2
2
  import { useOxy } from '@oxy.so/services/ui/client';
3
3
  import type { ChatMessage, ToolInvocation } from '../types';
4
4
  import { getTextFromContent, type MessageContent } from '../lib/content-utils';
@@ -7,7 +7,9 @@ import { agentMessageId, delegatedAgent } from '../lib/delegation';
7
7
  import { streamAliaChat } from '../lib/chat-transport';
8
8
  import type { AliaChatStreamEvent } from '../lib/chat-stream';
9
9
 
10
- const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
10
+ import { ALIA_API_URL } from '../lib/api-url';
11
+ import { useAliaActivityStore } from '../lib/activity-store';
12
+ import { turnActivityState } from '../lib/turn-state';
11
13
 
12
14
  export interface UseAliaChatOptions {
13
15
  /**
@@ -129,7 +131,7 @@ function waitWithAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T>
129
131
  * handler during its recorded compatibility window.
130
132
  */
131
133
  export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn {
132
- const { apiUrl = API_URL, model, clientContext, accessToken: accessTokenProp } = options;
134
+ const { apiUrl = ALIA_API_URL, model, clientContext, accessToken: accessTokenProp } = options;
133
135
  const directoryRef = useRef(new Map<string, { id: string; character?: unknown }>());
134
136
  directoryRef.current = new Map((options.agents ?? []).map((agent) => [agent.id, agent]));
135
137
  const requestUrl = '/v1/chat/completions';
@@ -147,6 +149,15 @@ export function useAliaChat(options: UseAliaChatOptions = {}): UseAliaChatReturn
147
149
  messagesRef.current = messages;
148
150
  }, [messages]);
149
151
 
152
+ // The turn's lifecycle, published for every "Ask Alia" button (`useAliaActivity`).
153
+ const activitySource = useId();
154
+ const activity = turnActivityState(messages, isStreaming);
155
+ useEffect(() => {
156
+ const { setLocal } = useAliaActivityStore.getState();
157
+ setLocal(activitySource, activity);
158
+ return () => setLocal(activitySource, 'idle');
159
+ }, [activitySource, activity]);
160
+
150
161
  const updateAssistant = useCallback(
151
162
  (assistantId: string, updater: (message: ChatMessage) => Partial<ChatMessage>) => {
152
163
  if (!mountedRef.current) return;
@@ -10,7 +10,7 @@ import {
10
10
  Easing,
11
11
  type SharedValue,
12
12
  } from 'react-native-reanimated';
13
- import { useSTTStore } from './useSpeechToText';
13
+ import { useDictationStore } from './useDictation';
14
14
  import { LEVEL_ATTACK_MS, LEVEL_RELEASE_MS } from '../lib/audio-level';
15
15
  import type { AgentState } from '../types';
16
16
 
@@ -95,8 +95,8 @@ export function useAmbientWave({
95
95
  // travel takes about three times as long. Tuning the constant moves both, by
96
96
  // different amounts. They are shared because one number should tune one
97
97
  // feel, not because the two are interchangeable.
98
- const sttIsRecording = useSTTStore((s) => s.isRecording);
99
- const sttMetering = useSTTStore((s) => s.metering);
98
+ const sttIsRecording = useDictationStore((s) => s.isRecording);
99
+ const sttMetering = useDictationStore((s) => s.metering);
100
100
  const stt = useSharedValue(0);
101
101
  useAnimatedReaction(
102
102
  () => (sttIsRecording ? Math.max(0.08, sttMetering) : -1),
@@ -0,0 +1,401 @@
1
+ import React from 'react';
2
+ import {
3
+ classifyIntake,
4
+ readFraction,
5
+ type IntakeKind,
6
+ type IntakeRefusal,
7
+ } from '../lib/attachment-intake';
8
+ import type { AliaAttachment } from '../types';
9
+
10
+ /**
11
+ * The composer's own file intake: the queue between "the user handed us a
12
+ * file" and "the attachment strip has it".
13
+ *
14
+ * ## There is no upload
15
+ *
16
+ * This is the first thing to know, and it decides everything below. Nothing in
17
+ * Alia uploads an attachment. A picked file becomes a `uri` and sits in the
18
+ * store until the message is sent, at which point the host's send
19
+ * inlines the images into the request body as `data:` URLs and drops the
20
+ * documents on the floor. There is no endpoint, no `FormData`, no XHR — so
21
+ * there are no bytes in flight to measure and nothing an abort could stop.
22
+ *
23
+ * What there IS, on web, is a read. A `File` that arrives by paste or by drop
24
+ * is a handle, not bytes, and it has to be read into a `data:` URL before it
25
+ * can be sent — a real transfer of real bytes, of a size the user chose, that
26
+ * takes real time for anything above a few megabytes. `FileReader` reports
27
+ * that read through `progress` events carrying `lengthComputable`, `loaded`
28
+ * and `total`, and it can be stopped with `abort()`. So the progress this
29
+ * queue shows is measured and the cancel button ends something: they are the
30
+ * read, honestly labelled, and not a stand-in for an upload that does not
31
+ * exist. When the browser declines to make the read measurable the tile says
32
+ * so by showing indeterminate activity — see `readFraction`.
33
+ *
34
+ * ## Why the queue is separate from the attachment list
35
+ *
36
+ * A file that is still being read is not an attachment yet: it has no `uri`,
37
+ * so `buildMessageContent` would silently skip it, and the strip would show a
38
+ * tile that contributes nothing to the message. It also cannot be REPAIRED in
39
+ * place — `PromptInput` only forwards an `onUpdateAttachment` when its consumer
40
+ * supplies one, and the chat page supplies `attachments`, `onAddAttachment`
41
+ * and `onRemoveAttachment` and no updater at all, so every `updateAttachment`
42
+ * from inside the composer lands in internal state that the controlled list
43
+ * shadows and nothing ever renders. (That is why the old paste path reached
44
+ * around the composer and called `useStore.getState().updateAttachment`
45
+ * directly.) Holding the in-flight files here instead means the strip's
46
+ * pending rows work the same whether the consumer wired an updater or not, and
47
+ * an attachment is only ever handed over once it has bytes behind it.
48
+ */
49
+
50
+ /** A file the composer is working on, has failed to read, or will not take. */
51
+ export interface IntakeItem {
52
+ id: string;
53
+ name: string;
54
+ size: number;
55
+ mimeType: string;
56
+ kind: IntakeKind;
57
+ /**
58
+ * `reading` while the bytes move, `failed` once they did not, `refused` for
59
+ * a file that never started — too big, or a dropped folder.
60
+ *
61
+ * A refusal is a queue entry rather than a toast for two reasons. It stays
62
+ * put: a drop of six files where one was too large is a sentence the user
63
+ * needs beside the five that worked, not one that slides away after four
64
+ * seconds. And it keeps the composer free of a notification dependency —
65
+ * `@oxy.so/bloom/toast` is a module the composer's own test suites do not
66
+ * stub, and a component that cannot be mounted in a test is a component
67
+ * whose behaviour nothing pins.
68
+ */
69
+ status: 'reading' | 'failed' | 'refused';
70
+ /**
71
+ * How far the read has got, 0–1 — or `null` for "no measurable progress",
72
+ * which the tile must render as indeterminate activity rather than as a
73
+ * number. Never a timer, never a guess.
74
+ */
75
+ fraction: number | null;
76
+ /** Set only on a `refused` row: which sentence to show. */
77
+ refusal?: IntakeRefusal;
78
+ }
79
+
80
+ /** A started read, and the one thing a caller can do to it. */
81
+ export interface ReadHandle {
82
+ abort: () => void;
83
+ }
84
+
85
+ export interface ReadHandlers {
86
+ onProgress: (fraction: number | null) => void;
87
+ onDone: (dataUrl: string) => void;
88
+ onFailed: () => void;
89
+ }
90
+
91
+ /**
92
+ * Begins a read and hands back the way to stop it. Injectable because
93
+ * `FileReader` is a browser API and the tests run in node — and because a test
94
+ * that cannot drive a read step by step cannot pin what cancel and retry do.
95
+ */
96
+ export type StartRead = (file: Blob, handlers: ReadHandlers) => ReadHandle;
97
+
98
+ /**
99
+ * The real reader: `FileReader`, the app's ONLY genuine cancellation point in
100
+ * the whole attachment path.
101
+ */
102
+ const startFileRead: StartRead = (file, handlers) => {
103
+ const reader = new FileReader();
104
+ reader.onprogress = (event) => handlers.onProgress(readFraction(event));
105
+ reader.onload = () => {
106
+ const result = reader.result;
107
+ // A `readAsDataURL` result is always a string when it succeeded; anything
108
+ // else means the read did not produce what the message needs, and a
109
+ // failure the user can retry is the truthful reading of that.
110
+ if (typeof result === 'string') handlers.onDone(result);
111
+ else handlers.onFailed();
112
+ };
113
+ reader.onerror = () => handlers.onFailed();
114
+ reader.readAsDataURL(file);
115
+ return { abort: () => reader.abort() };
116
+ };
117
+
118
+ export interface AttachmentIntake {
119
+ /** In-flight and failed files, in the order they arrived. */
120
+ items: IntakeItem[];
121
+ /** Take a batch of files from a paste or a drop. */
122
+ accept: (files: readonly File[]) => void;
123
+ /** Stop a read and drop the file. */
124
+ cancel: (id: string) => void;
125
+ /** Read a failed file again, from the file itself. */
126
+ retry: (id: string) => void;
127
+ /** Give up on a failed or refused file. */
128
+ dismiss: (id: string) => void;
129
+ /** True while anything is still being read — nothing may be sent yet. */
130
+ isBusy: boolean;
131
+ }
132
+
133
+ export interface AttachmentIntakeOptions {
134
+ addAttachment: (attachment: AliaAttachment) => void;
135
+ /**
136
+ * What the host's send can carry. Omitted, both; a file of a kind left out is refused
137
+ * (`unsupported`) instead of landing as a tile the send would drop.
138
+ */
139
+ kinds?: readonly IntakeKind[];
140
+ /** Swapped out by the tests; the real one is `startFileRead`. */
141
+ startRead?: StartRead;
142
+ }
143
+
144
+ const BOTH_KINDS: readonly IntakeKind[] = ['image', 'document'];
145
+
146
+ /** Ids that cannot collide, without a `Math.random()` anywhere near progress. */
147
+ let sequence = 0;
148
+ const nextId = (prefix: string) => `${prefix}-${Date.now()}-${++sequence}`;
149
+
150
+ export function useAttachmentIntake({
151
+ addAttachment,
152
+ kinds = BOTH_KINDS,
153
+ startRead = startFileRead,
154
+ }: AttachmentIntakeOptions): AttachmentIntake {
155
+ const [items, setItems] = React.useState<IntakeItem[]>([]);
156
+
157
+ /**
158
+ * The live reads and the files behind them, by item id.
159
+ *
160
+ * Refs rather than state, and deliberately so: a read handle is not
161
+ * something the strip draws, and re-rendering the composer every time a
162
+ * reader is registered would be a render per file per keystroke-free tick.
163
+ * More importantly `cancel` must reach the reader that is running NOW — a
164
+ * handle captured in a closure over state would be one render stale, which
165
+ * for a cancel button means aborting the read before last.
166
+ *
167
+ * The file is kept alongside for exactly one reason: retry has to re-read
168
+ * something, and the `File` is the only thing that still holds the bytes.
169
+ * Both maps are cleared the moment an item leaves the queue, which is what
170
+ * makes "release the reading resources" true rather than aspirational.
171
+ */
172
+ const reads = React.useRef(new Map<string, ReadHandle>());
173
+ const sources = React.useRef(new Map<string, File>());
174
+
175
+ const forget = React.useCallback((id: string) => {
176
+ reads.current.get(id)?.abort();
177
+ reads.current.delete(id);
178
+ sources.current.delete(id);
179
+ }, []);
180
+
181
+ const begin = React.useCallback(
182
+ (id: string, file: File, kind: IntakeKind) => {
183
+ /**
184
+ * Whether this read is still the one we are waiting on.
185
+ *
186
+ * `abort()` is a request, not a guarantee: a `FileReader` may still
187
+ * deliver a queued event afterwards, and some engines fire `load` for a
188
+ * read that had already completed in the background when the abort
189
+ * arrived. Without this check a cancelled file lands in the attachment
190
+ * list a beat after the user removed it — the row comes BACK — which is
191
+ * the failure mode that makes a cancel button worse than none.
192
+ *
193
+ * The map is the record of what is live, so asking it is also what makes
194
+ * "cancel" and "unmount" mean the same thing here.
195
+ */
196
+ const live = () => reads.current.has(id);
197
+
198
+ /*
199
+ * Registered BEFORE the read starts, then swapped for the real handle.
200
+ * A cached file can be read synchronously — the browser has the bytes
201
+ * already — and `onDone` would then run inside `startRead`, before there
202
+ * was anything in the map to say this read was live. The placeholder is
203
+ * what stops that completion being mistaken for a stale one and dropped.
204
+ */
205
+ const placeholder: ReadHandle = { abort: () => {} };
206
+ reads.current.set(id, placeholder);
207
+
208
+ const handle = startRead(file, {
209
+ onProgress: (fraction) => {
210
+ if (!live()) return;
211
+ setItems((prev) =>
212
+ prev.map((item) =>
213
+ item.id === id && item.status === 'reading' ? { ...item, fraction } : item,
214
+ ),
215
+ );
216
+ },
217
+ onDone: (dataUrl) => {
218
+ if (!live()) return;
219
+ // The read is over before the attachment exists, so the handle and
220
+ // the file go first: a completed reader kept in the map is a `File`
221
+ // kept alive, which for a 20 MiB drop is 20 MiB held for the life of
222
+ // the composer.
223
+ reads.current.delete(id);
224
+ sources.current.delete(id);
225
+ setItems((prev) => prev.filter((item) => item.id !== id));
226
+ addAttachment({
227
+ id,
228
+ uri: dataUrl,
229
+ type: kind,
230
+ name: file.name,
231
+ size: file.size,
232
+ mimeType: file.type || 'application/octet-stream',
233
+ });
234
+ },
235
+ onFailed: () => {
236
+ if (!live()) return;
237
+ // The reader is spent; the FILE is not, and retry needs it. This is
238
+ // the one path that keeps a source around after its read ended.
239
+ reads.current.delete(id);
240
+ setItems((prev) =>
241
+ prev.map((item) =>
242
+ item.id === id ? { ...item, status: 'failed', fraction: null } : item,
243
+ ),
244
+ );
245
+ },
246
+ });
247
+ // Only if the placeholder is still there. If the read already finished
248
+ // or failed inside the call above, the map has been moved on and writing
249
+ // the handle back would leave a spent reader registered as live.
250
+ if (reads.current.get(id) === placeholder) reads.current.set(id, handle);
251
+ },
252
+ [addAttachment, startRead],
253
+ );
254
+
255
+ const accept = React.useCallback(
256
+ (files: readonly File[]) => {
257
+ const queued: IntakeItem[] = [];
258
+ const starts: Array<() => void> = [];
259
+
260
+ for (const file of files) {
261
+ const name = file.name || '';
262
+ const verdict = classifyIntake({
263
+ name,
264
+ mimeType: file.type || '',
265
+ size: file.size,
266
+ });
267
+ const id = nextId('intake');
268
+ const refusal = !verdict.accepted
269
+ ? verdict.refusal
270
+ : kinds.includes(verdict.kind)
271
+ ? undefined
272
+ : ('unsupported' as const);
273
+ if (refusal !== undefined) {
274
+ // A file that silently fails to attach is the worst of the three
275
+ // outcomes: the user sends the message believing the picture went
276
+ // with it. The row names the file, because a drop is usually a batch
277
+ // and "one of them didn't work" is no answer when four were dropped.
278
+ queued.push({
279
+ id,
280
+ name: name || 'file',
281
+ size: file.size,
282
+ mimeType: file.type || '',
283
+ kind: 'document',
284
+ status: 'refused',
285
+ fraction: null,
286
+ refusal,
287
+ });
288
+ continue;
289
+ }
290
+ if (!verdict.accepted) continue;
291
+ if (verdict.kind === 'document') {
292
+ /*
293
+ * A document never gets read. `buildMessageContent` filters the
294
+ * attachment list down to `type === 'image'`, so a document's bytes
295
+ * go nowhere and reading them would be a progress bar over work that
296
+ * exists only to produce a string nobody consumes. The object URL is
297
+ * what the web pickers produce for the same file, it is instant, and
298
+ * the draft store gives it back when the tile is removed.
299
+ */
300
+ addAttachment({
301
+ id,
302
+ uri:
303
+ typeof URL !== 'undefined' && typeof URL.createObjectURL === 'function'
304
+ ? URL.createObjectURL(file)
305
+ : '',
306
+ type: 'document',
307
+ name: name || 'file',
308
+ size: file.size,
309
+ mimeType: file.type || 'application/octet-stream',
310
+ });
311
+ continue;
312
+ }
313
+
314
+ sources.current.set(id, file);
315
+ queued.push({
316
+ id,
317
+ name: name || 'image',
318
+ size: file.size,
319
+ mimeType: file.type || 'image/*',
320
+ kind: verdict.kind,
321
+ status: 'reading',
322
+ // Not zero. Zero is a claim that nothing has moved yet; `null` is the
323
+ // truth, which is that nothing has been reported yet.
324
+ fraction: null,
325
+ });
326
+ starts.push(() => begin(id, file, verdict.kind));
327
+ }
328
+
329
+ if (queued.length > 0) setItems((prev) => [...prev, ...queued]);
330
+ // Started after the rows exist, because a synchronous reader — the fake
331
+ // one in the tests, and a cached file in some browsers — can complete
332
+ // inside `startRead`, and a completion for a row that has not been added
333
+ // yet would be filtered against a list it is not in and then re-added by
334
+ // the batch above as a ghost that never finishes.
335
+ for (const start of starts) start();
336
+ },
337
+ [addAttachment, begin, kinds],
338
+ );
339
+
340
+ const cancel = React.useCallback(
341
+ (id: string) => {
342
+ forget(id);
343
+ setItems((prev) => prev.filter((item) => item.id !== id));
344
+ },
345
+ [forget],
346
+ );
347
+
348
+ const dismiss = cancel;
349
+
350
+ const retry = React.useCallback(
351
+ (id: string) => {
352
+ // The file IS the guard. A refused row never had one — nothing was
353
+ // read, and reading the same 30 MB again would reach the same verdict —
354
+ // and a cancelled one let go of it, so neither can start a second read
355
+ // by pressing a button the tile does not draw for them anyway.
356
+ const file = sources.current.get(id);
357
+ if (file === undefined) return;
358
+ setItems((prev) =>
359
+ prev.map((item) =>
360
+ item.id === id ? { ...item, status: 'reading', fraction: null } : item,
361
+ ),
362
+ );
363
+ // `"image"` is not an assumption: a document is handed over on arrival
364
+ // and never queues, so the only thing that can hold a source is a
365
+ // picture being read.
366
+ begin(id, file, 'image');
367
+ },
368
+ [begin],
369
+ );
370
+
371
+ /**
372
+ * Nothing survives the composer going away.
373
+ *
374
+ * An in-flight `FileReader` holds its `File`, and a `File` holds the bytes;
375
+ * a composer unmounted mid-read (navigating away, the drawer closing) would
376
+ * otherwise leave both running until the read finished into a `setItems` on
377
+ * a dead tree. Aborting is the whole of the fix and there is nowhere else to
378
+ * put it.
379
+ */
380
+ React.useEffect(() => {
381
+ const live = reads.current;
382
+ const held = sources.current;
383
+ return () => {
384
+ for (const handle of live.values()) handle.abort();
385
+ live.clear();
386
+ held.clear();
387
+ };
388
+ }, []);
389
+
390
+ return React.useMemo(
391
+ () => ({
392
+ items,
393
+ accept,
394
+ cancel,
395
+ retry,
396
+ dismiss,
397
+ isBusy: items.some((item) => item.status === 'reading'),
398
+ }),
399
+ [items, accept, cancel, retry, dismiss],
400
+ );
401
+ }
@@ -1,8 +1,8 @@
1
1
  /**
2
2
  * A voice call's live levels, as React state at ~20fps.
3
3
  *
4
- * `useVoiceRoom` writes the microphone level (from the recognizer) and the
5
- * playback level (from the clip being spoken) into a `VoiceLevelSource`; this
4
+ * `useVoiceRoom` writes the microphone level (from the captured buffers) and the
5
+ * playback level (from the answer being played) into a `VoiceLevelSource`; this
6
6
  * reads them on a clock so a wave can follow them without rendering once per
7
7
  * audio buffer. It used to put Web Audio analysers on a LiveKit room's tracks;
8
8
  * the signature kept its shape — `room` is now the level source `useVoiceRoom`