@alia.onl/sdk 7.2.8 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/README.md +234 -0
- package/package.json +9 -13
- package/src/components/AliaChatContent.tsx +0 -1
- package/src/components/AliaChatMessageList.tsx +0 -4
- package/src/components/ResearchProgressCard.tsx +3 -0
- package/src/components/ui/prompt-input/mic-button.tsx +2 -3
- package/src/components/ui/prompt-input/prompt-input.tsx +1 -4
- package/src/components/voice/VoiceControls.tsx +3 -54
- package/src/components/voice/VoiceSession.tsx +6 -19
- package/src/hooks/useAliaChat.ts +16 -1
- package/src/hooks/useAudioLevelMonitor.ts +26 -122
- package/src/hooks/useSpeechToText.ts +165 -150
- package/src/hooks/useTTS.ts +29 -29
- package/src/hooks/useVoiceRoom.ts +553 -409
- package/src/index.ts +0 -14
- package/src/lib/audio-level.ts +3 -2
- package/src/lib/chat-transport.ts +11 -0
- package/src/lib/speech-language.ts +20 -0
- package/src/lib/speech-messages.ts +31 -0
- package/src/lib/speech-recognition-types.ts +75 -0
- package/src/lib/speech-recognition.native.ts +171 -0
- package/src/lib/speech-recognition.ts +279 -0
- package/src/lib/speech-synthesis.ts +122 -0
- package/src/lib/voice-levels.ts +60 -0
- package/src/lib/voice-text.ts +192 -0
- package/src/lib/voice-turn.ts +87 -0
- package/src/types.ts +3 -3
- package/src/voice.ts +7 -3
package/src/index.ts
CHANGED
|
@@ -64,24 +64,10 @@ export type {
|
|
|
64
64
|
export { Button, buttonVariants, buttonTextVariants } from './components/ui/button';
|
|
65
65
|
export type { ButtonProps } from './components/ui/button';
|
|
66
66
|
export { Text, TextClassContext } from './components/ui/text';
|
|
67
|
-
export { ChatTextInput } from './components/ui/chat-text-input';
|
|
68
67
|
export { cn, formatFileSize } from './lib/utils';
|
|
69
68
|
export { getToolLabel, getToolActiveLabel, getResearchActiveLabel } from './lib/tool-registry';
|
|
70
69
|
export { getTextFromContent, getImagesFromContent } from './lib/content-utils';
|
|
71
70
|
|
|
72
|
-
// ── PromptInput ──
|
|
73
|
-
export { PromptInput } from './components/ui/prompt-input/prompt-input';
|
|
74
|
-
export type { PromptInputProps } from './components/ui/prompt-input/prompt-input';
|
|
75
|
-
export { usePromptInput, useIsFullscreen, PromptInputContext } from './components/ui/prompt-input/context';
|
|
76
|
-
export type { Attachment, Completion, PromptInputContextType } from './components/ui/prompt-input/context';
|
|
77
|
-
export { PromptInputTextarea } from './components/ui/prompt-input/textarea';
|
|
78
|
-
export { PromptInputActions } from './components/ui/prompt-input/actions';
|
|
79
|
-
export { PromptInputSubmitButton } from './components/ui/prompt-input/submit-button';
|
|
80
|
-
export { PromptInputMicButton } from './components/ui/prompt-input/mic-button';
|
|
81
|
-
export { PromptInputAddMenu } from './components/ui/prompt-input/add-menu';
|
|
82
|
-
export { PromptInputAttachments } from './components/ui/prompt-input/attachments';
|
|
83
|
-
export { PromptInputAutocomplete } from './components/ui/prompt-input/autocomplete';
|
|
84
|
-
|
|
85
71
|
// ── Picker hooks ──
|
|
86
72
|
export { useImagePicker } from './hooks/useImagePicker';
|
|
87
73
|
export type { ImagePickerAsset } from './hooks/useImagePicker';
|
package/src/lib/audio-level.ts
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
* How audio becomes the ambient field's 0..1 level — one curve, one pair of
|
|
3
3
|
* time constants, for every source that drives it.
|
|
4
4
|
*
|
|
5
|
-
* Dictation got here first and is the reference: the
|
|
6
|
-
*
|
|
5
|
+
* Dictation got here first and is the reference: the microphone's level is
|
|
6
|
+
* taken as dBFS (on web, from an analyser beside the recognizer —
|
|
7
|
+
* `speech-recognition.ts`), normalised against a -60 floor, and `useAmbientWave`
|
|
7
8
|
* smooths it with a fast attack and a slow decay before it reaches
|
|
8
9
|
* `AmbientField`. That is the behaviour people like, so it is the behaviour
|
|
9
10
|
* everything else adopts rather than re-invents — including "read aloud", which
|
|
@@ -18,6 +18,15 @@ export interface AliaChatRequest {
|
|
|
18
18
|
readonly url: string;
|
|
19
19
|
readonly model: string;
|
|
20
20
|
readonly messages: ReadonlyArray<{ readonly role: string; readonly content: string }>;
|
|
21
|
+
/**
|
|
22
|
+
* `'voice'` asks for an answer meant to be heard: short, conversational, no
|
|
23
|
+
* formatting (the API layers its voice response profile over the turn's
|
|
24
|
+
* prompt). Omitted for ordinary text chat, which sends exactly what it
|
|
25
|
+
* always sent.
|
|
26
|
+
*/
|
|
27
|
+
readonly responseMode?: 'voice';
|
|
28
|
+
/** The agent whose thread this turn belongs to, when there is one. */
|
|
29
|
+
readonly agentId?: string;
|
|
21
30
|
}
|
|
22
31
|
|
|
23
32
|
/**
|
|
@@ -42,6 +51,8 @@ export async function streamAliaChat(
|
|
|
42
51
|
model: request.model,
|
|
43
52
|
messages: request.messages,
|
|
44
53
|
stream: true,
|
|
54
|
+
...(request.responseMode === undefined ? {} : { responseMode: request.responseMode }),
|
|
55
|
+
...(request.agentId === undefined ? {} : { agentId: request.agentId }),
|
|
45
56
|
}),
|
|
46
57
|
signal,
|
|
47
58
|
});
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The language to recognize when the caller names none.
|
|
3
|
+
*
|
|
4
|
+
* The browser's own language on web; the runtime's resolved locale on native
|
|
5
|
+
* (Hermes implements `Intl`); `en-US` when neither answers. An app should pass
|
|
6
|
+
* its UI locale instead — the Alia app does — because a person who chose
|
|
7
|
+
* Spanish in the app expects to be understood in Spanish whatever the device
|
|
8
|
+
* is set to.
|
|
9
|
+
*/
|
|
10
|
+
export function defaultSpeechLanguage(): string {
|
|
11
|
+
const browser = globalThis.navigator?.language;
|
|
12
|
+
if (typeof browser === 'string' && browser !== '') return browser;
|
|
13
|
+
try {
|
|
14
|
+
const resolved = Intl.DateTimeFormat().resolvedOptions().locale;
|
|
15
|
+
if (resolved) return resolved;
|
|
16
|
+
} catch {
|
|
17
|
+
// No Intl on this runtime.
|
|
18
|
+
}
|
|
19
|
+
return 'en-US';
|
|
20
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { SpeechRecognitionFailure } from './speech-recognition-types';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The copy a person sees when listening fails — the same sentences dictation
|
|
5
|
+
* and voice mode used when they recorded through `expo-audio` and LiveKit, so
|
|
6
|
+
* moving recognition onto the device did not change what an error says.
|
|
7
|
+
*
|
|
8
|
+
* `null` for the failures that are not errors: `aborted` is a stop the caller
|
|
9
|
+
* asked for, and `no-speech` is a silence the caller recovers from.
|
|
10
|
+
*/
|
|
11
|
+
export function speechFailureMessage(failure: SpeechRecognitionFailure): string | null {
|
|
12
|
+
switch (failure.code) {
|
|
13
|
+
case 'aborted':
|
|
14
|
+
case 'no-speech':
|
|
15
|
+
return null;
|
|
16
|
+
case 'unsupported':
|
|
17
|
+
return 'Speech recognition is not available on this device or browser';
|
|
18
|
+
case 'not-allowed':
|
|
19
|
+
return 'Microphone permission required';
|
|
20
|
+
case 'audio-capture':
|
|
21
|
+
return failure.detail === 'NotReadableError' || failure.detail === 'TrackStartError'
|
|
22
|
+
? 'Microphone is in use by another application'
|
|
23
|
+
: 'No microphone found — check your input devices';
|
|
24
|
+
case 'network':
|
|
25
|
+
return 'Speech recognition needs a network connection';
|
|
26
|
+
case 'language-not-supported':
|
|
27
|
+
return 'Speech recognition does not support this language';
|
|
28
|
+
case 'other':
|
|
29
|
+
return 'Speech recognition failed';
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one shape on-device speech recognition has in this package.
|
|
3
|
+
*
|
|
4
|
+
* Two engines sit behind it and neither leaks through: the browser's Web
|
|
5
|
+
* Speech API (`speech-recognition.ts`, picked by web bundlers and by tests) and
|
|
6
|
+
* `expo-speech-recognition` on iOS and Android (`speech-recognition.native.ts`,
|
|
7
|
+
* picked by Metro's platform resolution). Dictation and the voice loop are
|
|
8
|
+
* written against this file only, so the audio never leaves the device through
|
|
9
|
+
* Alia — there is no transcription endpoint, and nothing here posts audio
|
|
10
|
+
* anywhere. Whether the ENGINE uses a network service is the platform's
|
|
11
|
+
* business and is stated in `docs/voice.mdx`.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
/** Why a recognition session failed, in the vocabulary both engines share. */
|
|
15
|
+
export type SpeechRecognitionFailureCode =
|
|
16
|
+
/** Microphone or speech-recognition permission was refused. */
|
|
17
|
+
| 'not-allowed'
|
|
18
|
+
/** No microphone, or it could not be opened. */
|
|
19
|
+
| 'audio-capture'
|
|
20
|
+
/** The engine needed the network and could not reach it. */
|
|
21
|
+
| 'network'
|
|
22
|
+
/** The engine does not recognize this language. */
|
|
23
|
+
| 'language-not-supported'
|
|
24
|
+
/** Nothing was said before the engine gave up. Recoverable: start again. */
|
|
25
|
+
| 'no-speech'
|
|
26
|
+
/** The session was aborted on purpose. Never an error to report. */
|
|
27
|
+
| 'aborted'
|
|
28
|
+
/** Recognition is not available on this device or in this browser. */
|
|
29
|
+
| 'unsupported'
|
|
30
|
+
/** Anything else — the engine was busy, interrupted, or said nothing useful. */
|
|
31
|
+
| 'other';
|
|
32
|
+
|
|
33
|
+
export interface SpeechRecognitionFailure {
|
|
34
|
+
readonly code: SpeechRecognitionFailureCode;
|
|
35
|
+
/** The engine's own words, for logs. Never shown as-is. */
|
|
36
|
+
readonly detail?: string;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface SpeechRecognitionHandlers {
|
|
40
|
+
/**
|
|
41
|
+
* The whole of what this session has heard so far, and whether the engine
|
|
42
|
+
* considers it settled. It is the FULL transcript each time, never a delta:
|
|
43
|
+
* both engines revise earlier words as more audio arrives.
|
|
44
|
+
*/
|
|
45
|
+
onResult(result: { transcript: string; isFinal: boolean }): void;
|
|
46
|
+
/** Input level, 0..1, on the same scale as dictation's metering. */
|
|
47
|
+
onLevel?(level: number): void;
|
|
48
|
+
onError(failure: SpeechRecognitionFailure): void;
|
|
49
|
+
/** The session is over, whichever way it ended. Always the last call. */
|
|
50
|
+
onEnd(): void;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface SpeechRecognitionOptions {
|
|
54
|
+
/** BCP-47 tag, e.g. `es-ES`. */
|
|
55
|
+
readonly lang: string;
|
|
56
|
+
/**
|
|
57
|
+
* Remove the device's own playback from what the microphone hears, where the
|
|
58
|
+
* platform can (iOS voice processing). Voice mode listens while it speaks,
|
|
59
|
+
* so without this it can hear itself.
|
|
60
|
+
*/
|
|
61
|
+
readonly echoCancellation?: boolean;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export interface SpeechRecognitionSession {
|
|
65
|
+
/** Stop listening and deliver a final result, then `onEnd`. */
|
|
66
|
+
stop(): void;
|
|
67
|
+
/** Stop listening and discard; `onEnd` still follows. */
|
|
68
|
+
abort(): void;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Map a raw 0..1-ish reading to the field's level, clamped. */
|
|
72
|
+
export function clampLevel(level: number): number {
|
|
73
|
+
if (!Number.isFinite(level)) return 0;
|
|
74
|
+
return Math.min(1, Math.max(0, level));
|
|
75
|
+
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* On-device speech recognition for iOS and Android, through
|
|
3
|
+
* `expo-speech-recognition` (Apple's Speech framework, Android's
|
|
4
|
+
* `SpeechRecognizer`). Metro picks this file over `speech-recognition.ts` on
|
|
5
|
+
* native; the contract is `speech-recognition-types.ts`.
|
|
6
|
+
*
|
|
7
|
+
* `expo-speech-recognition` is a peer of the SDK and a native module: an app
|
|
8
|
+
* must depend on it (the root entry's dictation reaches this file) and list its
|
|
9
|
+
* config plugin (which writes `NSSpeechRecognitionUsageDescription`,
|
|
10
|
+
* `NSMicrophoneUsageDescription` and Android's `RECORD_AUDIO` plus the
|
|
11
|
+
* recognizer-package visibility query), then ship a new native build.
|
|
12
|
+
*
|
|
13
|
+
* The module is a process-wide singleton — one recognition at a time — so the
|
|
14
|
+
* hooks never start a session before the previous one has reported `onEnd`. A
|
|
15
|
+
* late `end` from an aborted session would otherwise land on its successor's
|
|
16
|
+
* listeners.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import type { ExpoSpeechRecognitionErrorCode, ExpoSpeechRecognitionModule as NativeModule } from 'expo-speech-recognition';
|
|
20
|
+
import {
|
|
21
|
+
clampLevel,
|
|
22
|
+
type SpeechRecognitionFailure,
|
|
23
|
+
type SpeechRecognitionFailureCode,
|
|
24
|
+
type SpeechRecognitionHandlers,
|
|
25
|
+
type SpeechRecognitionOptions,
|
|
26
|
+
type SpeechRecognitionSession,
|
|
27
|
+
} from './speech-recognition-types';
|
|
28
|
+
|
|
29
|
+
type SpeechModule = typeof NativeModule;
|
|
30
|
+
|
|
31
|
+
let loaded: SpeechModule | null | undefined;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The native module, loaded on first use — never at import.
|
|
35
|
+
*
|
|
36
|
+
* `expo-speech-recognition` calls `requireNativeModule` when it is imported,
|
|
37
|
+
* which THROWS in a binary built before the module was added (an older dev
|
|
38
|
+
* client, Expo Go). This file is reachable from the SDK's root entry through
|
|
39
|
+
* dictation, so a top-level import would take the whole app down in such a
|
|
40
|
+
* build over a feature nobody had touched yet. Loaded here, the same build
|
|
41
|
+
* answers "not available" and everything else keeps working.
|
|
42
|
+
*/
|
|
43
|
+
function speechModule(): SpeechModule | null {
|
|
44
|
+
if (loaded !== undefined) return loaded;
|
|
45
|
+
try {
|
|
46
|
+
loaded = (require('expo-speech-recognition') as typeof import('expo-speech-recognition'))
|
|
47
|
+
.ExpoSpeechRecognitionModule;
|
|
48
|
+
} catch {
|
|
49
|
+
loaded = null;
|
|
50
|
+
}
|
|
51
|
+
return loaded;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function isSpeechRecognitionAvailable(): boolean {
|
|
55
|
+
try {
|
|
56
|
+
return speechModule()?.isRecognitionAvailable() ?? false;
|
|
57
|
+
} catch {
|
|
58
|
+
return false;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export async function requestSpeechRecognitionPermission(): Promise<SpeechRecognitionFailure | null> {
|
|
63
|
+
const module = speechModule();
|
|
64
|
+
if (module === null || !isSpeechRecognitionAvailable()) return { code: 'unsupported' };
|
|
65
|
+
try {
|
|
66
|
+
const permission = await module.requestPermissionsAsync();
|
|
67
|
+
return permission.granted ? null : { code: 'not-allowed', detail: permission.status };
|
|
68
|
+
} catch (error: unknown) {
|
|
69
|
+
return { code: 'other', detail: error instanceof Error ? error.message : 'permission request failed' };
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function failureOf(error: ExpoSpeechRecognitionErrorCode): SpeechRecognitionFailureCode {
|
|
74
|
+
switch (error) {
|
|
75
|
+
case 'not-allowed':
|
|
76
|
+
case 'service-not-allowed':
|
|
77
|
+
return 'not-allowed';
|
|
78
|
+
case 'audio-capture':
|
|
79
|
+
return 'audio-capture';
|
|
80
|
+
case 'network':
|
|
81
|
+
return 'network';
|
|
82
|
+
case 'language-not-supported':
|
|
83
|
+
return 'language-not-supported';
|
|
84
|
+
case 'no-speech':
|
|
85
|
+
case 'speech-timeout':
|
|
86
|
+
return 'no-speech';
|
|
87
|
+
case 'aborted':
|
|
88
|
+
return 'aborted';
|
|
89
|
+
default:
|
|
90
|
+
return 'other';
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/** `volumechange` is -2..10, and anything below 0 is inaudible. */
|
|
95
|
+
const VOLUME_CEILING = 10;
|
|
96
|
+
|
|
97
|
+
export function startSpeechRecognition(
|
|
98
|
+
options: SpeechRecognitionOptions,
|
|
99
|
+
handlers: SpeechRecognitionHandlers,
|
|
100
|
+
): SpeechRecognitionSession {
|
|
101
|
+
const module = speechModule();
|
|
102
|
+
if (module === null) {
|
|
103
|
+
handlers.onError({ code: 'unsupported' });
|
|
104
|
+
handlers.onEnd();
|
|
105
|
+
return { stop: () => undefined, abort: () => undefined };
|
|
106
|
+
}
|
|
107
|
+
const ExpoSpeechRecognitionModule = module;
|
|
108
|
+
let ended = false;
|
|
109
|
+
/**
|
|
110
|
+
* Finals already delivered. Android's continuous mode starts a new segment
|
|
111
|
+
* after every final result, so the utterance is the settled segments plus
|
|
112
|
+
* whatever is still in flight; iOS only finalizes on stop and always sends
|
|
113
|
+
* the whole transcript, so this stays empty there.
|
|
114
|
+
*/
|
|
115
|
+
let settled = '';
|
|
116
|
+
|
|
117
|
+
const subscriptions = [
|
|
118
|
+
ExpoSpeechRecognitionModule.addListener('result', (event) => {
|
|
119
|
+
if (ended) return;
|
|
120
|
+
const current = event.results[0]?.transcript ?? '';
|
|
121
|
+
const transcript = `${settled} ${current}`.replace(/\s+/g, ' ').trim();
|
|
122
|
+
if (event.isFinal) settled = transcript;
|
|
123
|
+
handlers.onResult({ transcript, isFinal: event.isFinal });
|
|
124
|
+
}),
|
|
125
|
+
ExpoSpeechRecognitionModule.addListener('error', (event) => {
|
|
126
|
+
if (ended) return;
|
|
127
|
+
const code = failureOf(event.error);
|
|
128
|
+
if (code === 'aborted') return;
|
|
129
|
+
handlers.onError({ code, detail: event.message });
|
|
130
|
+
}),
|
|
131
|
+
ExpoSpeechRecognitionModule.addListener('end', () => finish()),
|
|
132
|
+
ExpoSpeechRecognitionModule.addListener('volumechange', (event) => {
|
|
133
|
+
if (ended) return;
|
|
134
|
+
handlers.onLevel?.(clampLevel(event.value / VOLUME_CEILING));
|
|
135
|
+
}),
|
|
136
|
+
];
|
|
137
|
+
|
|
138
|
+
function finish(): void {
|
|
139
|
+
if (ended) return;
|
|
140
|
+
ended = true;
|
|
141
|
+
for (const subscription of subscriptions) subscription.remove();
|
|
142
|
+
handlers.onLevel?.(0);
|
|
143
|
+
handlers.onEnd();
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
try {
|
|
147
|
+
ExpoSpeechRecognitionModule.start({
|
|
148
|
+
lang: options.lang,
|
|
149
|
+
interimResults: true,
|
|
150
|
+
continuous: true,
|
|
151
|
+
addsPunctuation: true,
|
|
152
|
+
maxAlternatives: 1,
|
|
153
|
+
volumeChangeEventOptions: { enabled: handlers.onLevel !== undefined, intervalMillis: 100 },
|
|
154
|
+
...(options.echoCancellation ? { iosVoiceProcessingEnabled: true } : {}),
|
|
155
|
+
});
|
|
156
|
+
} catch (error: unknown) {
|
|
157
|
+
if (!ended) handlers.onError({ code: 'other', detail: error instanceof Error ? error.message : 'start failed' });
|
|
158
|
+
finish();
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
return {
|
|
162
|
+
stop() {
|
|
163
|
+
if (ended) return;
|
|
164
|
+
ExpoSpeechRecognitionModule.stop();
|
|
165
|
+
},
|
|
166
|
+
abort() {
|
|
167
|
+
if (ended) return;
|
|
168
|
+
ExpoSpeechRecognitionModule.abort();
|
|
169
|
+
},
|
|
170
|
+
};
|
|
171
|
+
}
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* On-device speech recognition for the web: the browser's Web Speech API.
|
|
3
|
+
*
|
|
4
|
+
* Metro resolves `speech-recognition.native.ts` on iOS and Android, so this
|
|
5
|
+
* file is what a web bundle — and every test — gets. See
|
|
6
|
+
* `speech-recognition-types.ts` for the contract both implement.
|
|
7
|
+
*
|
|
8
|
+
* ## Why a microphone stream is opened beside the recognizer
|
|
9
|
+
*
|
|
10
|
+
* The Web Speech API reports words, not sound: there is no level event, and a
|
|
11
|
+
* refused or missing microphone surfaces as a bare `not-allowed` or
|
|
12
|
+
* `audio-capture` with nothing to tell "blocked" from "busy". Opening the
|
|
13
|
+
* microphone first with `getUserMedia` answers both. It is where the browser
|
|
14
|
+
* asks for permission (once — the recognizer shares the origin's grant), its
|
|
15
|
+
* DOMException names say exactly what is wrong, and an `AnalyserNode` on the
|
|
16
|
+
* stream is the honest level the dictation trace and the voice field move to.
|
|
17
|
+
* The stream is never recorded or sent anywhere.
|
|
18
|
+
*
|
|
19
|
+
* ## Support
|
|
20
|
+
*
|
|
21
|
+
* Chromium browsers and Safari implement it (Chrome as `webkitSpeechRecognition`
|
|
22
|
+
* and through a Google speech service; Safari through Apple's). Firefox does
|
|
23
|
+
* not, and `isSpeechRecognitionAvailable()` says so before anything asks for a
|
|
24
|
+
* microphone.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { levelFromDbfs } from './audio-level';
|
|
28
|
+
import type {
|
|
29
|
+
SpeechRecognitionFailure,
|
|
30
|
+
SpeechRecognitionFailureCode,
|
|
31
|
+
SpeechRecognitionHandlers,
|
|
32
|
+
SpeechRecognitionOptions,
|
|
33
|
+
SpeechRecognitionSession,
|
|
34
|
+
} from './speech-recognition-types';
|
|
35
|
+
|
|
36
|
+
// ── The slice of the Web Speech API this uses ──
|
|
37
|
+
// Declared here rather than taken from lib.dom: TypeScript's DOM library does
|
|
38
|
+
// not ship the prefixed constructor, and the unprefixed one is not everywhere.
|
|
39
|
+
|
|
40
|
+
interface WebSpeechAlternative { readonly transcript: string }
|
|
41
|
+
interface WebSpeechResult { readonly isFinal: boolean; readonly length: number; readonly [index: number]: WebSpeechAlternative }
|
|
42
|
+
interface WebSpeechResultList { readonly length: number; readonly [index: number]: WebSpeechResult }
|
|
43
|
+
interface WebSpeechResultEvent { readonly results: WebSpeechResultList }
|
|
44
|
+
interface WebSpeechErrorEvent { readonly error: string; readonly message?: string }
|
|
45
|
+
|
|
46
|
+
interface WebSpeechRecognition {
|
|
47
|
+
lang: string;
|
|
48
|
+
continuous: boolean;
|
|
49
|
+
interimResults: boolean;
|
|
50
|
+
maxAlternatives: number;
|
|
51
|
+
onresult: ((event: WebSpeechResultEvent) => void) | null;
|
|
52
|
+
onerror: ((event: WebSpeechErrorEvent) => void) | null;
|
|
53
|
+
onend: (() => void) | null;
|
|
54
|
+
start(): void;
|
|
55
|
+
stop(): void;
|
|
56
|
+
abort(): void;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
type WebSpeechRecognitionConstructor = new () => WebSpeechRecognition;
|
|
60
|
+
|
|
61
|
+
function recognitionConstructor(): WebSpeechRecognitionConstructor | null {
|
|
62
|
+
const scope = globalThis as unknown as {
|
|
63
|
+
SpeechRecognition?: WebSpeechRecognitionConstructor;
|
|
64
|
+
webkitSpeechRecognition?: WebSpeechRecognitionConstructor;
|
|
65
|
+
};
|
|
66
|
+
return scope.SpeechRecognition ?? scope.webkitSpeechRecognition ?? null;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Whether this browser can recognize speech at all. Firefox cannot. */
|
|
70
|
+
export function isSpeechRecognitionAvailable(): boolean {
|
|
71
|
+
return recognitionConstructor() !== null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Ask for the microphone before a session needs it.
|
|
76
|
+
*
|
|
77
|
+
* The browser asks once per origin; `startSpeechRecognition` opens the stream
|
|
78
|
+
* itself, so this is only for callers that want the answer up front (voice mode
|
|
79
|
+
* does, so a refusal ends the call before it starts rather than on the first
|
|
80
|
+
* word).
|
|
81
|
+
*/
|
|
82
|
+
export async function requestSpeechRecognitionPermission(): Promise<SpeechRecognitionFailure | null> {
|
|
83
|
+
if (!isSpeechRecognitionAvailable()) return { code: 'unsupported' };
|
|
84
|
+
const media = globalThis.navigator?.mediaDevices;
|
|
85
|
+
if (media?.getUserMedia === undefined) return null;
|
|
86
|
+
try {
|
|
87
|
+
const stream = await media.getUserMedia({ audio: true });
|
|
88
|
+
for (const track of stream.getTracks()) track.stop();
|
|
89
|
+
return null;
|
|
90
|
+
} catch (error: unknown) {
|
|
91
|
+
return mediaFailure(error);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function domErrorName(error: unknown): string {
|
|
96
|
+
// A DOMException is not always `instanceof Error`, so read `.name` off the object.
|
|
97
|
+
if (typeof error === 'object' && error !== null && 'name' in error && typeof error.name === 'string') {
|
|
98
|
+
return error.name;
|
|
99
|
+
}
|
|
100
|
+
return '';
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function mediaFailure(error: unknown): SpeechRecognitionFailure {
|
|
104
|
+
const name = domErrorName(error);
|
|
105
|
+
switch (name) {
|
|
106
|
+
case 'NotAllowedError':
|
|
107
|
+
case 'PermissionDeniedError':
|
|
108
|
+
case 'SecurityError':
|
|
109
|
+
return { code: 'not-allowed', detail: name };
|
|
110
|
+
case 'NotFoundError':
|
|
111
|
+
case 'DevicesNotFoundError':
|
|
112
|
+
case 'OverconstrainedError':
|
|
113
|
+
case 'NotReadableError':
|
|
114
|
+
case 'TrackStartError':
|
|
115
|
+
return { code: 'audio-capture', detail: name };
|
|
116
|
+
default:
|
|
117
|
+
return { code: 'other', detail: name || 'getUserMedia failed' };
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function recognitionFailure(event: WebSpeechErrorEvent): SpeechRecognitionFailure {
|
|
122
|
+
const map: Record<string, SpeechRecognitionFailureCode> = {
|
|
123
|
+
'not-allowed': 'not-allowed',
|
|
124
|
+
'service-not-allowed': 'not-allowed',
|
|
125
|
+
'audio-capture': 'audio-capture',
|
|
126
|
+
network: 'network',
|
|
127
|
+
'language-not-supported': 'language-not-supported',
|
|
128
|
+
'no-speech': 'no-speech',
|
|
129
|
+
aborted: 'aborted',
|
|
130
|
+
};
|
|
131
|
+
return { code: map[event.error] ?? 'other', detail: event.error };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Everything heard so far, as one string: Chrome segments carry their own leading spaces. */
|
|
135
|
+
function transcriptOf(results: WebSpeechResultList): { transcript: string; isFinal: boolean } {
|
|
136
|
+
let transcript = '';
|
|
137
|
+
let isFinal = results.length > 0;
|
|
138
|
+
for (let index = 0; index < results.length; index += 1) {
|
|
139
|
+
const result = results[index];
|
|
140
|
+
if (result === undefined) continue;
|
|
141
|
+
transcript += result[0]?.transcript ?? '';
|
|
142
|
+
if (!result.isFinal) isFinal = false;
|
|
143
|
+
}
|
|
144
|
+
return { transcript: transcript.replace(/\s+/g, ' ').trim(), isFinal };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const LEVEL_INTERVAL_MS = 100;
|
|
148
|
+
|
|
149
|
+
/** An analyser on the microphone, polled on a clock. Returns its teardown. */
|
|
150
|
+
function meterStream(stream: MediaStream, onLevel: (level: number) => void): () => void {
|
|
151
|
+
const AudioContextCtor = (globalThis as unknown as { AudioContext?: typeof AudioContext }).AudioContext;
|
|
152
|
+
if (AudioContextCtor === undefined) return () => undefined;
|
|
153
|
+
let context: AudioContext;
|
|
154
|
+
try {
|
|
155
|
+
context = new AudioContextCtor();
|
|
156
|
+
} catch {
|
|
157
|
+
return () => undefined;
|
|
158
|
+
}
|
|
159
|
+
if (context.state === 'suspended') context.resume().catch(() => undefined);
|
|
160
|
+
const analyser = context.createAnalyser();
|
|
161
|
+
analyser.fftSize = 512;
|
|
162
|
+
context.createMediaStreamSource(stream).connect(analyser);
|
|
163
|
+
const buffer = new Float32Array(analyser.fftSize);
|
|
164
|
+
const timer = setInterval(() => {
|
|
165
|
+
analyser.getFloatTimeDomainData(buffer);
|
|
166
|
+
let sum = 0;
|
|
167
|
+
for (let index = 0; index < buffer.length; index += 1) sum += buffer[index] * buffer[index];
|
|
168
|
+
const rms = Math.sqrt(sum / buffer.length);
|
|
169
|
+
onLevel(rms > 0 ? levelFromDbfs(20 * Math.log10(rms)) : 0);
|
|
170
|
+
}, LEVEL_INTERVAL_MS);
|
|
171
|
+
return () => {
|
|
172
|
+
clearInterval(timer);
|
|
173
|
+
context.close().catch(() => undefined);
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Listen until stopped.
|
|
179
|
+
*
|
|
180
|
+
* Continuous with interim results: the caller decides when an utterance is
|
|
181
|
+
* over (voice mode on a silence timer, dictation on the stop button), because
|
|
182
|
+
* the browser's own end-of-speech detection cuts people off mid-thought.
|
|
183
|
+
*/
|
|
184
|
+
export function startSpeechRecognition(
|
|
185
|
+
options: SpeechRecognitionOptions,
|
|
186
|
+
handlers: SpeechRecognitionHandlers,
|
|
187
|
+
): SpeechRecognitionSession {
|
|
188
|
+
let recognition: WebSpeechRecognition | null = null;
|
|
189
|
+
let stream: MediaStream | null = null;
|
|
190
|
+
let stopMeter: (() => void) | null = null;
|
|
191
|
+
let ended = false;
|
|
192
|
+
let stopRequested: 'stop' | 'abort' | null = null;
|
|
193
|
+
|
|
194
|
+
const release = (): void => {
|
|
195
|
+
stopMeter?.();
|
|
196
|
+
stopMeter = null;
|
|
197
|
+
if (stream !== null) for (const track of stream.getTracks()) track.stop();
|
|
198
|
+
stream = null;
|
|
199
|
+
};
|
|
200
|
+
|
|
201
|
+
const finish = (): void => {
|
|
202
|
+
if (ended) return;
|
|
203
|
+
ended = true;
|
|
204
|
+
release();
|
|
205
|
+
handlers.onLevel?.(0);
|
|
206
|
+
handlers.onEnd();
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
const fail = (failure: SpeechRecognitionFailure): void => {
|
|
210
|
+
if (ended) return;
|
|
211
|
+
handlers.onError(failure);
|
|
212
|
+
finish();
|
|
213
|
+
};
|
|
214
|
+
|
|
215
|
+
void (async () => {
|
|
216
|
+
const Recognition = recognitionConstructor();
|
|
217
|
+
if (Recognition === null) {
|
|
218
|
+
fail({ code: 'unsupported' });
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
const media = globalThis.navigator?.mediaDevices;
|
|
223
|
+
if (media?.getUserMedia !== undefined) {
|
|
224
|
+
try {
|
|
225
|
+
stream = await media.getUserMedia({
|
|
226
|
+
audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
|
|
227
|
+
});
|
|
228
|
+
} catch (error: unknown) {
|
|
229
|
+
fail(mediaFailure(error));
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
if (stopRequested !== null) {
|
|
233
|
+
finish();
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
if (handlers.onLevel !== undefined) stopMeter = meterStream(stream, handlers.onLevel);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
const instance = new Recognition();
|
|
240
|
+
recognition = instance;
|
|
241
|
+
instance.lang = options.lang;
|
|
242
|
+
instance.continuous = true;
|
|
243
|
+
instance.interimResults = true;
|
|
244
|
+
instance.maxAlternatives = 1;
|
|
245
|
+
instance.onresult = (event) => {
|
|
246
|
+
if (ended) return;
|
|
247
|
+
handlers.onResult(transcriptOf(event.results));
|
|
248
|
+
};
|
|
249
|
+
instance.onerror = (event) => {
|
|
250
|
+
if (ended) return;
|
|
251
|
+
const failure = recognitionFailure(event);
|
|
252
|
+
if (failure.code === 'aborted') return;
|
|
253
|
+
handlers.onError(failure);
|
|
254
|
+
};
|
|
255
|
+
instance.onend = finish;
|
|
256
|
+
try {
|
|
257
|
+
instance.start();
|
|
258
|
+
} catch (error: unknown) {
|
|
259
|
+
fail({ code: 'other', detail: domErrorName(error) || 'start failed' });
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
if (stopRequested === 'stop') instance.stop();
|
|
263
|
+
if (stopRequested === 'abort') instance.abort();
|
|
264
|
+
})();
|
|
265
|
+
|
|
266
|
+
return {
|
|
267
|
+
stop() {
|
|
268
|
+
if (ended) return;
|
|
269
|
+
stopRequested = 'stop';
|
|
270
|
+
recognition?.stop();
|
|
271
|
+
},
|
|
272
|
+
abort() {
|
|
273
|
+
if (ended) return;
|
|
274
|
+
stopRequested = 'abort';
|
|
275
|
+
if (recognition === null) return;
|
|
276
|
+
recognition.abort();
|
|
277
|
+
},
|
|
278
|
+
};
|
|
279
|
+
}
|