@alia.onl/sdk 13.1.0 → 15.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +190 -0
- package/README.md +83 -20
- package/package.json +8 -4
- package/src/components/AliaAvatarIcon.tsx +30 -0
- package/src/components/AliaChatContent.tsx +57 -107
- package/src/components/AliaChatMessageList.tsx +2 -10
- package/src/components/AliaChatScreen.tsx +28 -25
- package/src/components/AliaChatSheet.tsx +30 -8
- package/src/components/AliaComposer.tsx +382 -0
- package/src/components/ComposerDropZone.tsx +235 -0
- package/src/components/voice/VoiceControls.tsx +28 -4
- package/src/components/voice/VoiceSession.tsx +23 -7
- package/src/hooks/useAliaActivity.ts +31 -0
- package/src/hooks/useAliaChat.ts +14 -3
- package/src/hooks/useAmbientWave.ts +3 -3
- package/src/hooks/useAttachmentIntake.ts +401 -0
- package/src/hooks/useAudioLevelMonitor.ts +2 -2
- package/src/hooks/useDictation.ts +295 -0
- package/src/hooks/useFocusWhenPresented.ts +41 -0
- package/src/hooks/usePendingQuestions.ts +19 -0
- package/src/hooks/useTTS.ts +24 -63
- package/src/hooks/useVoiceRoom.ts +148 -671
- package/src/index.ts +19 -3
- package/src/lib/activity-socket.ts +102 -0
- package/src/lib/activity-store.ts +53 -0
- package/src/lib/api-url.ts +2 -0
- package/src/lib/attachment-intake.ts +154 -0
- package/src/lib/attachment-tiles.ts +74 -0
- package/src/lib/audio-level.ts +2 -2
- package/src/lib/dictation-recorder-types.ts +37 -0
- package/src/lib/dictation-recorder.native.ts +90 -0
- package/src/lib/dictation-recorder.ts +131 -0
- package/src/lib/dictation.ts +208 -0
- package/src/lib/labels.tsx +50 -1
- package/src/lib/pcm.ts +57 -0
- package/src/lib/speech-language.ts +1 -1
- package/src/lib/speech-messages.ts +43 -65
- package/src/lib/speech-synthesis.ts +104 -9
- package/src/lib/turn-state.ts +15 -0
- package/src/lib/voice-audio-engine-types.ts +59 -0
- package/src/lib/voice-audio-engine.native.ts +232 -0
- package/src/lib/voice-audio-engine.ts +350 -0
- package/src/lib/voice-call.ts +539 -0
- package/src/lib/voice-levels.ts +7 -3
- package/src/lib/voice-protocol.ts +309 -0
- package/src/tools.ts +1 -0
- package/src/types.ts +63 -4
- package/src/voice.ts +31 -10
- package/tsconfig.json +2 -0
- package/src/hooks/useSpeechToText.ts +0 -268
- package/src/lib/speech-recognition-types.ts +0 -108
- package/src/lib/speech-recognition.native.ts +0 -237
- package/src/lib/speech-recognition.ts +0 -308
- package/src/lib/voice-text.ts +0 -198
- package/src/lib/voice-turn.ts +0 -89
package/src/index.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
export type { AliaChatAgent, AliaChatState } from './components/AliaChatContent';
|
|
7
7
|
export { AliaChatSheet } from './components/AliaChatSheet';
|
|
8
8
|
export type { AliaChatSheetProps, AliaChatSheetRef } from './components/AliaChatSheet';
|
|
9
|
+
export type { AliaTurn } from './components/AliaChatContent';
|
|
9
10
|
export { AliaChatScreen } from './components/AliaChatScreen';
|
|
10
11
|
export type { AliaChatScreenProps } from './components/AliaChatScreen';
|
|
11
12
|
|
|
@@ -19,6 +20,9 @@ export type { AmbientFieldProps } from './components/AmbientField';
|
|
|
19
20
|
export { ALIA_AVATAR, AliaAvatar } from './components/AliaAvatar';
|
|
20
21
|
export { DEFAULT_AGENT_CHARACTER, readAgentCharacter } from './lib/agent-character';
|
|
21
22
|
export type { AliaAvatarProps, AliaAvatarState } from './components/AliaAvatar';
|
|
23
|
+
export { AliaAvatarIcon } from './components/AliaAvatarIcon';
|
|
24
|
+
export { useAliaActivity } from './hooks/useAliaActivity';
|
|
25
|
+
export type { UseAliaActivityOptions } from './hooks/useAliaActivity';
|
|
22
26
|
export { IdentityMark } from './components/IdentityMark';
|
|
23
27
|
export type { IdentityMarkProps, IdentityMarkState } from './components/IdentityMark';
|
|
24
28
|
export { AliaMarkdown } from './components/Markdown';
|
|
@@ -26,7 +30,14 @@ export type { AliaMarkdownProps, RenderCodeBlock } from './components/Markdown';
|
|
|
26
30
|
export { AliaWelcomeMessage } from './components/AliaWelcomeMessage';
|
|
27
31
|
export { PlanPreviewCard } from './components/PlanPreviewCard';
|
|
28
32
|
export { AskedQuestions, QuestionCard } from './components/QuestionCard';
|
|
33
|
+
export { AliaComposer } from './components/AliaComposer';
|
|
34
|
+
export type {
|
|
35
|
+
AliaComposerProps,
|
|
36
|
+
AliaComposerQuestions,
|
|
37
|
+
AliaComposerSuggestions,
|
|
38
|
+
} from './components/AliaComposer';
|
|
29
39
|
export { ComposerSuggestions } from './components/ComposerSuggestions';
|
|
40
|
+
export { usePendingQuestions } from './hooks/usePendingQuestions';
|
|
30
41
|
export { useComposerSuggestions, useSuggestionQuery } from './hooks/useComposerSuggestions';
|
|
31
42
|
export { composerCompletions } from './lib/suggestions';
|
|
32
43
|
export type { Completion } from './lib/suggestions';
|
|
@@ -44,13 +55,17 @@ export type { UseAliaChatOptions, UseAliaChatReturn } from './hooks/useAliaChat'
|
|
|
44
55
|
export { AliaChatStreamError } from './lib/chat-stream';
|
|
45
56
|
export type { AliaChatFailure } from './lib/chat-stream';
|
|
46
57
|
|
|
47
|
-
// ──
|
|
58
|
+
// ── Read aloud and dictation (no call, no audio engine) ──
|
|
48
59
|
export { useTTS } from './hooks/useTTS';
|
|
49
60
|
export type { UseTTSOptions } from './hooks/useTTS';
|
|
50
|
-
export {
|
|
61
|
+
export { useDictation, useDictationStore } from './hooks/useDictation';
|
|
62
|
+
export type { UseDictationOptions, DictationState } from './hooks/useDictation';
|
|
63
|
+
export { DictationError, requestTranscription } from './lib/dictation';
|
|
64
|
+
export type { DictationErrorCode, TranscriptionResult } from './lib/dictation';
|
|
51
65
|
export { VOICE_ERROR_MESSAGES } from './lib/speech-messages';
|
|
52
66
|
export type { VoiceErrorCode } from './lib/speech-messages';
|
|
53
|
-
export
|
|
67
|
+
export { SpeechClipError } from './lib/speech-synthesis';
|
|
68
|
+
export type { SpeechClipErrorCode } from './lib/speech-synthesis';
|
|
54
69
|
|
|
55
70
|
// ── Model catalogue (GET /catalogue) ──
|
|
56
71
|
// Exported so a consumer building its own picker reads the real models the
|
|
@@ -68,6 +83,7 @@ export type { Catalogue, CatalogueEntry, ModelSelection, ReasoningEffort } from
|
|
|
68
83
|
export type {
|
|
69
84
|
ChatMessage,
|
|
70
85
|
ToolInvocation,
|
|
86
|
+
AliaAttachment,
|
|
71
87
|
AliaSuggestion,
|
|
72
88
|
ResearchProgress,
|
|
73
89
|
ResearchSource,
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { io, type Socket } from 'socket.io-client';
|
|
2
|
+
import type { AliaAvatarState } from '../components/AliaAvatar';
|
|
3
|
+
import { useAliaActivityStore } from './activity-store';
|
|
4
|
+
|
|
5
|
+
/** A run that goes quiet this long is over: a crashed run never says `idle`. */
|
|
6
|
+
export const ACTIVITY_QUIET_MS = 90_000;
|
|
7
|
+
/** Between tries after the server refuses a handshake (socket.io does not retry those itself). */
|
|
8
|
+
const RETRY_MS = 5_000;
|
|
9
|
+
|
|
10
|
+
const STATES: ReadonlySet<string> = new Set<AliaAvatarState>([
|
|
11
|
+
'idle',
|
|
12
|
+
'thinking',
|
|
13
|
+
'working',
|
|
14
|
+
'writing',
|
|
15
|
+
]);
|
|
16
|
+
|
|
17
|
+
interface Connection {
|
|
18
|
+
socket: Socket;
|
|
19
|
+
refs: number;
|
|
20
|
+
quiet: Map<string, ReturnType<typeof setTimeout>>;
|
|
21
|
+
retry: ReturnType<typeof setTimeout> | null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** One connection per API origin, however many buttons ask for it. */
|
|
25
|
+
const connections = new Map<string, Connection>();
|
|
26
|
+
|
|
27
|
+
function open(url: string, getToken: () => string | null): Connection {
|
|
28
|
+
const socket = io(url, {
|
|
29
|
+
transports: ['websocket'],
|
|
30
|
+
// Read on every (re)connect, so a rotated access token is the one that goes out.
|
|
31
|
+
auth: (cb) => cb({ token: getToken() }),
|
|
32
|
+
reconnection: true,
|
|
33
|
+
reconnectionAttempts: Infinity,
|
|
34
|
+
reconnectionDelay: 1000,
|
|
35
|
+
reconnectionDelayMax: 10_000,
|
|
36
|
+
});
|
|
37
|
+
const connection: Connection = { socket, refs: 0, quiet: new Map(), retry: null };
|
|
38
|
+
const store = useAliaActivityStore.getState;
|
|
39
|
+
|
|
40
|
+
const reset = (): void => {
|
|
41
|
+
for (const timer of connection.quiet.values()) clearTimeout(timer);
|
|
42
|
+
connection.quiet.clear();
|
|
43
|
+
store().clearRemote();
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
socket.on('connect', () => socket.emit('subscribe-alia-activity'));
|
|
47
|
+
// What happened while disconnected is unknown; a stale "working" is worse than idle.
|
|
48
|
+
socket.on('disconnect', reset);
|
|
49
|
+
socket.on('connect_error', () => {
|
|
50
|
+
// A refused handshake (no token yet, an expired one) ends the socket's own retrying.
|
|
51
|
+
if (socket.active || connection.retry !== null) return;
|
|
52
|
+
connection.retry = setTimeout(() => {
|
|
53
|
+
connection.retry = null;
|
|
54
|
+
socket.connect();
|
|
55
|
+
}, RETRY_MS);
|
|
56
|
+
});
|
|
57
|
+
socket.on('alia:activity', (event: unknown) => {
|
|
58
|
+
if (typeof event !== 'object' || event === null) return;
|
|
59
|
+
const { state, sessionId } = event as { state?: unknown; sessionId?: unknown };
|
|
60
|
+
if (typeof sessionId !== 'string' || typeof state !== 'string' || !STATES.has(state)) return;
|
|
61
|
+
const pending = connection.quiet.get(sessionId);
|
|
62
|
+
if (pending !== undefined) clearTimeout(pending);
|
|
63
|
+
connection.quiet.delete(sessionId);
|
|
64
|
+
store().setRemote(sessionId, state as AliaAvatarState);
|
|
65
|
+
if (state !== 'idle') {
|
|
66
|
+
connection.quiet.set(
|
|
67
|
+
sessionId,
|
|
68
|
+
setTimeout(() => {
|
|
69
|
+
connection.quiet.delete(sessionId);
|
|
70
|
+
store().setRemote(sessionId, 'idle');
|
|
71
|
+
}, ACTIVITY_QUIET_MS),
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
});
|
|
75
|
+
return connection;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Listen for Alia's activity on the person's behalf. Every caller shares the one connection for
|
|
80
|
+
* `url`; the last to release closes it. Returns the release.
|
|
81
|
+
*/
|
|
82
|
+
export function acquireActivitySocket(url: string, getToken: () => string | null): () => void {
|
|
83
|
+
let held = connections.get(url);
|
|
84
|
+
if (!held) {
|
|
85
|
+
held = open(url, getToken);
|
|
86
|
+
connections.set(url, held);
|
|
87
|
+
}
|
|
88
|
+
const connection = held;
|
|
89
|
+
connection.refs++;
|
|
90
|
+
let released = false;
|
|
91
|
+
return () => {
|
|
92
|
+
if (released) return;
|
|
93
|
+
released = true;
|
|
94
|
+
if (--connection.refs > 0) return;
|
|
95
|
+
connections.delete(url);
|
|
96
|
+
if (connection.retry !== null) clearTimeout(connection.retry);
|
|
97
|
+
connection.socket.removeAllListeners();
|
|
98
|
+
connection.socket.disconnect();
|
|
99
|
+
for (const timer of connection.quiet.values()) clearTimeout(timer);
|
|
100
|
+
useAliaActivityStore.getState().clearRemote();
|
|
101
|
+
};
|
|
102
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { create } from 'zustand';
|
|
2
|
+
import type { AliaAvatarState } from '../components/AliaAvatar';
|
|
3
|
+
|
|
4
|
+
type Sources = Readonly<Record<string, AliaAvatarState>>;
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* What Alia is doing, from every place that can know. Two sources feed it and neither owns it:
|
|
8
|
+
*
|
|
9
|
+
* - `local`: the turns running in THIS app, one entry per `useAliaChat` instance (the sheet);
|
|
10
|
+
* - `remote`: runs the server reports for the person, one entry per agent session, so a button
|
|
11
|
+
* reacts to work that is not in any open chat.
|
|
12
|
+
*
|
|
13
|
+
* An idle source holds no entry, so neither map grows with use.
|
|
14
|
+
*/
|
|
15
|
+
interface AliaActivityStore {
|
|
16
|
+
local: Sources;
|
|
17
|
+
remote: Sources;
|
|
18
|
+
setLocal(source: string, state: AliaAvatarState): void;
|
|
19
|
+
setRemote(session: string, state: AliaAvatarState): void;
|
|
20
|
+
clearRemote(): void;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function put(map: Sources, key: string, state: AliaAvatarState): Sources {
|
|
24
|
+
if (state === 'idle') {
|
|
25
|
+
if (!(key in map)) return map;
|
|
26
|
+
const { [key]: _gone, ...rest } = map;
|
|
27
|
+
return rest;
|
|
28
|
+
}
|
|
29
|
+
return map[key] === state ? map : { ...map, [key]: state };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export const useAliaActivityStore = create<AliaActivityStore>((set) => ({
|
|
33
|
+
local: {},
|
|
34
|
+
remote: {},
|
|
35
|
+
setLocal: (source, state) => set((s) => ({ local: put(s.local, source, state) })),
|
|
36
|
+
setRemote: (session, state) => set((s) => ({ remote: put(s.remote, session, state) })),
|
|
37
|
+
clearRemote: () => set((s) => (Object.keys(s.remote).length === 0 ? s : { remote: {} })),
|
|
38
|
+
}));
|
|
39
|
+
|
|
40
|
+
/** The most telling thing any source is doing: a tool call outranks thinking, thinking outranks writing. */
|
|
41
|
+
const RANK: Record<AliaAvatarState, number> = { idle: 0, writing: 1, thinking: 2, working: 3 };
|
|
42
|
+
|
|
43
|
+
export function busiest(...maps: ReadonlyArray<Sources>): AliaAvatarState {
|
|
44
|
+
let top: AliaAvatarState = 'idle';
|
|
45
|
+
for (const map of maps) {
|
|
46
|
+
for (const state of Object.values(map)) if (RANK[state] > RANK[top]) top = state;
|
|
47
|
+
}
|
|
48
|
+
return top;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** A primitive, so a subscriber re-renders when the state changes and not when a session does. */
|
|
52
|
+
export const selectAliaActivity = (s: AliaActivityStore): AliaAvatarState =>
|
|
53
|
+
busiest(s.local, s.remote);
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What the composer will take in, and how far a file has actually got.
|
|
3
|
+
*
|
|
4
|
+
* Alia had no answer to either question. A file reached the attachment strip
|
|
5
|
+
* by exactly one route — a picker that hands back a `uri` and asks nothing —
|
|
6
|
+
* so there was no size it would refuse, no kind it would separate, and, once
|
|
7
|
+
* drag-and-drop existed, no shared place to state those rules for a second
|
|
8
|
+
* route. Worse, a drag is a stateful gesture: the browser fires `dragenter`
|
|
9
|
+
* and `dragleave` for every element the pointer crosses, so a drop affordance
|
|
10
|
+
* driven by a boolean flickers off the instant the pointer moves from the bar
|
|
11
|
+
* onto the textarea INSIDE it. The counter that fixes that is four lines of
|
|
12
|
+
* arithmetic guarding a visible effect, which is precisely the kind of thing
|
|
13
|
+
* that is easy to get subtly wrong and almost impossible to watch going wrong.
|
|
14
|
+
*
|
|
15
|
+
* So it lives here, beside `turn-selection.ts`, under the same rule: nothing
|
|
16
|
+
* in this file may import react-native, read a store, or touch the DOM. The
|
|
17
|
+
* reader that moves the bytes is a platform adapter and lives with the
|
|
18
|
+
* composer; what can be decided by arithmetic is decided here, where it can be
|
|
19
|
+
* checked without mounting a composer or staging a drag.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The largest file the composer will take.
|
|
24
|
+
*
|
|
25
|
+
* An attached image is not uploaded anywhere — the app's `attachment-utils.ts`
|
|
26
|
+
* inlines it into the request as a `data:` URL — so its bytes are carried in
|
|
27
|
+
* memory as a base64 string roughly a third larger than the file, twice over
|
|
28
|
+
* while the request is built. 20 MiB is the point past which that stops being
|
|
29
|
+
* a message and starts being a reason the tab dies.
|
|
30
|
+
*/
|
|
31
|
+
export const MAX_ATTACHMENT_BYTES = 20 * 1024 * 1024;
|
|
32
|
+
|
|
33
|
+
export type IntakeKind = 'image' | 'document';
|
|
34
|
+
|
|
35
|
+
/** Why a file was turned away. Each one has its own sentence to the user. */
|
|
36
|
+
export type IntakeRefusal = 'empty' | 'too-large' | 'unsupported';
|
|
37
|
+
|
|
38
|
+
export type IntakeVerdict =
|
|
39
|
+
| { accepted: true; kind: IntakeKind }
|
|
40
|
+
| { accepted: false; refusal: IntakeRefusal };
|
|
41
|
+
|
|
42
|
+
export interface IntakeCandidate {
|
|
43
|
+
name: string;
|
|
44
|
+
mimeType: string;
|
|
45
|
+
size: number;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Extensions that name a picture when the browser will not.
|
|
50
|
+
*
|
|
51
|
+
* A dragged file's `type` is filled in from the OS, and it comes through empty
|
|
52
|
+
* often enough to matter — from an archive manager, from a file manager with a
|
|
53
|
+
* thin mime database, from anything that drops a path rather than a handle.
|
|
54
|
+
* Falling back to the name is not a nicety: without it a dropped `.png` is
|
|
55
|
+
* filed as a document, and a document is never inlined into the request at all.
|
|
56
|
+
*/
|
|
57
|
+
const IMAGE_EXTENSIONS = ['png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp', 'avif', 'heic', 'heif'];
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Whether a candidate is a picture, by what the browser says first and by its
|
|
61
|
+
* name only when the browser said nothing.
|
|
62
|
+
*/
|
|
63
|
+
export function intakeKind(candidate: IntakeCandidate): IntakeKind {
|
|
64
|
+
if (candidate.mimeType.startsWith('image/')) return 'image';
|
|
65
|
+
const extension = candidate.name.split('.').pop()?.toLowerCase() ?? '';
|
|
66
|
+
return IMAGE_EXTENSIONS.includes(extension) ? 'image' : 'document';
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* The one gate both web routes pass through — pasted files and dropped ones —
|
|
71
|
+
* so a rule cannot hold for one and not the other.
|
|
72
|
+
*
|
|
73
|
+
* The zero-byte refusal is not pedantry about empty files. Dropping a FOLDER
|
|
74
|
+
* on web yields a `File` with size 0 and no type, in every browser; without
|
|
75
|
+
* this the composer accepts a tile named after a directory whose contents it
|
|
76
|
+
* will never have, and whose read completes with nothing in it.
|
|
77
|
+
*/
|
|
78
|
+
export function classifyIntake(candidate: IntakeCandidate): IntakeVerdict {
|
|
79
|
+
if (candidate.size <= 0) return { accepted: false, refusal: 'empty' };
|
|
80
|
+
if (candidate.size > MAX_ATTACHMENT_BYTES) return { accepted: false, refusal: 'too-large' };
|
|
81
|
+
return { accepted: true, kind: intakeKind(candidate) };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** The four things that can happen to the count of drags over the composer. */
|
|
85
|
+
export type DragStep = 'enter' | 'leave' | 'drop' | 'reset';
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* How many dragged-over elements deep the pointer is.
|
|
89
|
+
*
|
|
90
|
+
* `dragenter` and `dragleave` fire per ELEMENT, not per target, and they fire
|
|
91
|
+
* in that order: moving from the bar onto the textarea inside it produces
|
|
92
|
+
* `dragenter` (textarea) then `dragleave` (bar). A boolean set by enter and
|
|
93
|
+
* cleared by leave therefore ends that move switched OFF while the pointer is
|
|
94
|
+
* still over the composer, and the affordance strobes as the pointer crosses
|
|
95
|
+
* each child on the way in. Counting elements instead is the standard answer,
|
|
96
|
+
* and the depth only reaches zero when the pointer has genuinely left
|
|
97
|
+
* everything.
|
|
98
|
+
*
|
|
99
|
+
* `drop` resets rather than decrements: the browser does not send the matching
|
|
100
|
+
* `dragleave` for the element the file landed on, so a decrement leaves the
|
|
101
|
+
* count stuck at one and the overlay pinned over the composer for good.
|
|
102
|
+
*/
|
|
103
|
+
export function nextDragDepth(depth: number, step: DragStep): number {
|
|
104
|
+
switch (step) {
|
|
105
|
+
case 'enter':
|
|
106
|
+
return depth + 1;
|
|
107
|
+
case 'leave':
|
|
108
|
+
// Clamped, because a `dragleave` can arrive without its `dragenter` — a
|
|
109
|
+
// drag that began before the listener was attached, or one whose
|
|
110
|
+
// `dragenter` was swallowed by an element that stopped propagation. A
|
|
111
|
+
// negative depth would then need two full entries before the overlay
|
|
112
|
+
// came back.
|
|
113
|
+
return Math.max(0, depth - 1);
|
|
114
|
+
case 'drop':
|
|
115
|
+
case 'reset':
|
|
116
|
+
return 0;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Whether this drag is carrying files at all.
|
|
122
|
+
*
|
|
123
|
+
* `DataTransfer.types` contains the literal `"Files"` when it is; dragging
|
|
124
|
+
* selected text or a link gives `"text/plain"` and `"text/uri-list"` instead.
|
|
125
|
+
* Without the question the composer lights up when the user drags a word
|
|
126
|
+
* across it, and then swallows the drop.
|
|
127
|
+
*/
|
|
128
|
+
export function dragCarriesFiles(types: readonly string[] | undefined): boolean {
|
|
129
|
+
return types !== undefined && types.includes('Files');
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* How far a read has got, as a fraction — or `null` when nothing measurable
|
|
134
|
+
* was reported.
|
|
135
|
+
*
|
|
136
|
+
* `null` is the important return, and it is why this is a function rather than
|
|
137
|
+
* a division at the call site. #608 §7: "Cuando no hay avance medible, mostrar
|
|
138
|
+
* actividad indeterminada. No inventar porcentajes." A `ProgressEvent` whose
|
|
139
|
+
* `lengthComputable` is false carries a `total` of 0, and dividing by it gives
|
|
140
|
+
* `NaN` or `Infinity` — both of which render as SOMETHING once handed to a
|
|
141
|
+
* bar, and that something is invented. Bloom's `use-attachment-queue.ts`
|
|
142
|
+
* invents its number outright, from a `Math.random()` ramp on a 50ms tick;
|
|
143
|
+
* nothing in this file may grow a timer.
|
|
144
|
+
*/
|
|
145
|
+
export function readFraction(progress: {
|
|
146
|
+
lengthComputable: boolean;
|
|
147
|
+
loaded: number;
|
|
148
|
+
total: number;
|
|
149
|
+
}): number | null {
|
|
150
|
+
if (!progress.lengthComputable) return null;
|
|
151
|
+
if (!Number.isFinite(progress.total) || progress.total <= 0) return null;
|
|
152
|
+
if (!Number.isFinite(progress.loaded) || progress.loaded < 0) return null;
|
|
153
|
+
return Math.min(1, progress.loaded / progress.total);
|
|
154
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import type { ComposerPanelAttachment } from '@oxy.so/bloom/composer-panel';
|
|
2
|
+
import type { IntakeItem } from '../hooks/useAttachmentIntake';
|
|
3
|
+
import type { AliaAttachment } from '../types';
|
|
4
|
+
import { MAX_ATTACHMENT_BYTES } from './attachment-intake';
|
|
5
|
+
import type { AliaChatLabels } from './labels';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The composer's tiles: landed files, then the ones the intake still holds.
|
|
9
|
+
*
|
|
10
|
+
* A file whose read failed, or that was refused, stays in the strip with the
|
|
11
|
+
* sentence that says why — a drop of six where one did not make it is a
|
|
12
|
+
* sentence the user needs beside the five that did, not a toast that slides
|
|
13
|
+
* away. `retryable` marks the ones a second read could fix: a failed read,
|
|
14
|
+
* whose `File` the intake kept. A refusal is not: the same 30 MB file reaches
|
|
15
|
+
* the same verdict — and since Bloom's retry button is the panel's, drawn on
|
|
16
|
+
* every error tile once `onAttachmentRetry` is set, the composer shows a
|
|
17
|
+
* refusal as a toast rather than as a tile it would offer to retry.
|
|
18
|
+
*/
|
|
19
|
+
export interface ComposerTile extends ComposerPanelAttachment {
|
|
20
|
+
error?: string;
|
|
21
|
+
retryable?: boolean;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** The words a tile and a refusal use. */
|
|
25
|
+
type TileLabels = Pick<
|
|
26
|
+
AliaChatLabels,
|
|
27
|
+
'composerUntitled' | 'fileReadFailed' | 'fileTooLarge' | 'fileEmpty' | 'fileUnsupported'
|
|
28
|
+
>;
|
|
29
|
+
|
|
30
|
+
/** Why an intake item is not an attachment, or `undefined` while it is being read. */
|
|
31
|
+
export function intakeError(item: IntakeItem, labels: TileLabels): string | undefined {
|
|
32
|
+
const name = item.name || labels.composerUntitled;
|
|
33
|
+
switch (item.status) {
|
|
34
|
+
case 'reading':
|
|
35
|
+
return undefined;
|
|
36
|
+
case 'failed':
|
|
37
|
+
return labels.fileReadFailed(name);
|
|
38
|
+
case 'refused':
|
|
39
|
+
if (item.refusal === 'unsupported') return labels.fileUnsupported(name);
|
|
40
|
+
return item.refusal === 'too-large'
|
|
41
|
+
? labels.fileTooLarge(name, `${Math.round(MAX_ATTACHMENT_BYTES / (1024 * 1024))} MB`)
|
|
42
|
+
: labels.fileEmpty(name);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export function composerTiles(
|
|
47
|
+
attachments: readonly AliaAttachment[],
|
|
48
|
+
items: readonly IntakeItem[],
|
|
49
|
+
labels: TileLabels,
|
|
50
|
+
): ComposerTile[] {
|
|
51
|
+
return [
|
|
52
|
+
...attachments.map((a) => ({
|
|
53
|
+
id: a.id,
|
|
54
|
+
name: a.name,
|
|
55
|
+
kind: a.type === 'image' ? ('image' as const) : ('document' as const),
|
|
56
|
+
src: a.type === 'image' ? a.uri : undefined,
|
|
57
|
+
})),
|
|
58
|
+
...items.map((item): ComposerTile => {
|
|
59
|
+
const tile: ComposerTile = {
|
|
60
|
+
id: item.id,
|
|
61
|
+
name: item.name || labels.composerUntitled,
|
|
62
|
+
kind: item.kind === 'image' ? 'image' : 'document',
|
|
63
|
+
};
|
|
64
|
+
if (item.status === 'reading') {
|
|
65
|
+
// Real progress only; an unmeasured read is an empty ring, not a guess.
|
|
66
|
+
tile.progress = item.fraction === null ? 0 : Math.round(item.fraction * 100);
|
|
67
|
+
return tile;
|
|
68
|
+
}
|
|
69
|
+
tile.error = intakeError(item, labels);
|
|
70
|
+
tile.retryable = item.status === 'failed';
|
|
71
|
+
return tile;
|
|
72
|
+
}),
|
|
73
|
+
];
|
|
74
|
+
}
|
package/src/lib/audio-level.ts
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
* time constants, for every source that drives it.
|
|
4
4
|
*
|
|
5
5
|
* Dictation got here first and is the reference: the microphone's level is
|
|
6
|
-
* taken as dBFS (on web, from an analyser beside the
|
|
7
|
-
* `
|
|
6
|
+
* taken as dBFS (on web, from an analyser beside the recorder —
|
|
7
|
+
* `dictation-recorder.ts`), normalised against a -60 floor, and `useAmbientWave`
|
|
8
8
|
* smooths it with a fast attack and a slow decay before it reaches
|
|
9
9
|
* `AmbientField`. That is the behaviour people like, so it is the behaviour
|
|
10
10
|
* everything else adopts rather than re-invents — including "read aloud", which
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { DictationClip } from './dictation';
|
|
2
|
+
|
|
3
|
+
/** Why the microphone could not record. */
|
|
4
|
+
export type DictationRecorderErrorCode = 'mic-denied' | 'mic-unavailable';
|
|
5
|
+
|
|
6
|
+
export class DictationRecorderError extends Error {
|
|
7
|
+
readonly code: DictationRecorderErrorCode;
|
|
8
|
+
|
|
9
|
+
constructor(code: DictationRecorderErrorCode, message?: string) {
|
|
10
|
+
super(message ?? code);
|
|
11
|
+
this.name = 'DictationRecorderError';
|
|
12
|
+
this.code = code;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* A clip recorder for the composer's microphone:
|
|
18
|
+
*
|
|
19
|
+
* - web (`dictation-recorder.ts`): `MediaRecorder`, `audio/webm;codecs=opus`
|
|
20
|
+
* or `audio/mp4` (Safari), level from an `AnalyserNode` on the same stream;
|
|
21
|
+
* - iOS / Android (`dictation-recorder.native.ts`): the `expo-audio`
|
|
22
|
+
* recorder, `RecordingPresets.HIGH_QUALITY` (m4a/AAC → `audio/mp4`) with
|
|
23
|
+
* metering on.
|
|
24
|
+
*
|
|
25
|
+
* Levels are the real recorder's, 0..1 on the dictation meter's dBFS scale.
|
|
26
|
+
*/
|
|
27
|
+
export interface DictationRecorder {
|
|
28
|
+
/**
|
|
29
|
+
* Ask for the microphone and start recording; resolves once the recorder
|
|
30
|
+
* says it started. Rejects with a {@link DictationRecorderError}.
|
|
31
|
+
*/
|
|
32
|
+
start(onLevel: (level: number) => void): Promise<void>;
|
|
33
|
+
/** Stop and hand over the clip, or `null` when nothing was recorded. */
|
|
34
|
+
stop(): Promise<DictationClip | null>;
|
|
35
|
+
/** Stop and throw the clip away. */
|
|
36
|
+
cancel(): void;
|
|
37
|
+
}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dictation's iOS / Android recorder: the `expo-audio` recorder (already a
|
|
3
|
+
* peer of this SDK), `RecordingPresets.HIGH_QUALITY` in mono with metering on.
|
|
4
|
+
* The clip is m4a/AAC, sent as `audio/mp4`. No new native module: the call's
|
|
5
|
+
* engine (`react-native-audio-api`) is not needed to record a clip.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { useMemo } from 'react';
|
|
9
|
+
import {
|
|
10
|
+
RecordingPresets,
|
|
11
|
+
requestRecordingPermissionsAsync,
|
|
12
|
+
setAudioModeAsync,
|
|
13
|
+
useAudioRecorder,
|
|
14
|
+
} from 'expo-audio';
|
|
15
|
+
import { levelFromDbfs } from './audio-level';
|
|
16
|
+
import type { DictationClip } from './dictation';
|
|
17
|
+
import { DictationRecorderError, type DictationRecorder } from './dictation-recorder-types';
|
|
18
|
+
|
|
19
|
+
export * from './dictation-recorder-types';
|
|
20
|
+
|
|
21
|
+
const LEVEL_POLL_MS = 100;
|
|
22
|
+
|
|
23
|
+
const OPTIONS = { ...RecordingPresets.HIGH_QUALITY, numberOfChannels: 1, isMeteringEnabled: true };
|
|
24
|
+
|
|
25
|
+
export function useDictationRecorder(): DictationRecorder {
|
|
26
|
+
const recorder = useAudioRecorder(OPTIONS);
|
|
27
|
+
|
|
28
|
+
return useMemo(() => {
|
|
29
|
+
let levelTimer: ReturnType<typeof setInterval> | null = null;
|
|
30
|
+
let lastDurationMs = 0;
|
|
31
|
+
let recording = false;
|
|
32
|
+
|
|
33
|
+
const stopLevels = (): void => {
|
|
34
|
+
if (levelTimer !== null) clearInterval(levelTimer);
|
|
35
|
+
levelTimer = null;
|
|
36
|
+
};
|
|
37
|
+
const releaseSession = (): void => {
|
|
38
|
+
void setAudioModeAsync({ allowsRecording: false }).catch(() => undefined);
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
return {
|
|
42
|
+
async start(onLevel) {
|
|
43
|
+
const permission = await requestRecordingPermissionsAsync().catch(() => null);
|
|
44
|
+
if (!permission?.granted) throw new DictationRecorderError('mic-denied');
|
|
45
|
+
try {
|
|
46
|
+
await setAudioModeAsync({ allowsRecording: true, playsInSilentMode: true });
|
|
47
|
+
await recorder.prepareToRecordAsync();
|
|
48
|
+
recorder.record();
|
|
49
|
+
} catch {
|
|
50
|
+
releaseSession();
|
|
51
|
+
throw new DictationRecorderError('mic-unavailable');
|
|
52
|
+
}
|
|
53
|
+
recording = true;
|
|
54
|
+
lastDurationMs = 0;
|
|
55
|
+
levelTimer = setInterval(() => {
|
|
56
|
+
const status = recorder.getStatus();
|
|
57
|
+
lastDurationMs = status.durationMillis;
|
|
58
|
+
onLevel(typeof status.metering === 'number' ? levelFromDbfs(status.metering) : 0);
|
|
59
|
+
}, LEVEL_POLL_MS);
|
|
60
|
+
},
|
|
61
|
+
|
|
62
|
+
async stop(): Promise<DictationClip | null> {
|
|
63
|
+
if (!recording) return null;
|
|
64
|
+
recording = false;
|
|
65
|
+
stopLevels();
|
|
66
|
+
const durationMs = recorder.getStatus().durationMillis || lastDurationMs;
|
|
67
|
+
try {
|
|
68
|
+
await recorder.stop();
|
|
69
|
+
} finally {
|
|
70
|
+
releaseSession();
|
|
71
|
+
}
|
|
72
|
+
const uri = recorder.uri;
|
|
73
|
+
if (!uri) return null;
|
|
74
|
+
return {
|
|
75
|
+
file: { uri, name: 'dictation.m4a', type: 'audio/mp4' },
|
|
76
|
+
mediaType: 'audio/mp4',
|
|
77
|
+
durationMs,
|
|
78
|
+
};
|
|
79
|
+
},
|
|
80
|
+
|
|
81
|
+
cancel(): void {
|
|
82
|
+
if (!recording) return;
|
|
83
|
+
recording = false;
|
|
84
|
+
stopLevels();
|
|
85
|
+
void recorder.stop().catch(() => undefined);
|
|
86
|
+
releaseSession();
|
|
87
|
+
},
|
|
88
|
+
};
|
|
89
|
+
}, [recorder]);
|
|
90
|
+
}
|