@adea-ai/app-framework 0.47.0 → 0.48.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/framework-source.json +1 -1
- package/dist/voice/voice-build.d.ts +62 -0
- package/dist/voice/voice-build.js +247 -0
- package/dist/voice/voice-client.d.ts +130 -0
- package/dist/voice/voice-client.js +405 -0
- package/dist/voice/voice-controls.d.ts +43 -0
- package/dist/voice/voice-controls.js +313 -0
- package/dist/voice/voice-host.d.ts +29 -0
- package/dist/voice/voice-host.js +248 -0
- package/dist/voice/voice-protocol.d.ts +127 -0
- package/dist/voice/voice-protocol.js +120 -0
- package/dist/voice/voice-settings.d.ts +15 -0
- package/dist/voice/voice-settings.js +1095 -0
- package/dist/voice/voice.d.ts +5 -0
- package/dist/voice/voice.js +5 -0
- package/package.json +21 -4
- package/src/voice/voice-client.ts +579 -0
- package/src/voice/voice-controls.tsx +354 -0
- package/src/voice/voice-protocol.ts +189 -0
- package/src/voice/voice-settings.tsx +879 -0
- package/src/voice/voice.ts +13 -0
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** The renderer's voice API: the client, the service names, the settings section and controls. */
|
|
2
|
+
export * from './voice-client';
|
|
3
|
+
export { EVENTS_WAIT_MS, MAX_SPEAK_CHARACTERS, VOICE_SERVICES, type VoiceEvent, type VoiceEventBatch, type VoiceServiceName, type VoiceSpeechVoice, } from './voice-protocol';
|
|
4
|
+
export * from './voice-settings';
|
|
5
|
+
export * from './voice-controls';
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { createDesktopVoiceClient, createWebVoiceClient } from "./voice-client.js";
|
|
2
|
+
import { EVENTS_WAIT_MS, MAX_SPEAK_CHARACTERS, VOICE_SERVICES } from "./voice-protocol.js";
|
|
3
|
+
import { VOICE_COMMON_LOCALES, VoiceSettingsPanel, voiceSettingsSection } from "./voice-settings.js";
|
|
4
|
+
import { DictationButton, ReadAloudButton, createDictation } from "./voice-controls.js";
|
|
5
|
+
export { DictationButton, EVENTS_WAIT_MS, MAX_SPEAK_CHARACTERS, ReadAloudButton, VOICE_COMMON_LOCALES, VOICE_SERVICES, VoiceSettingsPanel, createDesktopVoiceClient, createDictation, createWebVoiceClient, voiceSettingsSection };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adea-ai/app-framework",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.48.1",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -48,7 +48,10 @@
|
|
|
48
48
|
"README.md",
|
|
49
49
|
"LICENSE",
|
|
50
50
|
"NOTICE",
|
|
51
|
-
"templates"
|
|
51
|
+
"templates",
|
|
52
|
+
"src/voice",
|
|
53
|
+
"!src/voice/voice-host.ts",
|
|
54
|
+
"!src/voice/voice-build.ts"
|
|
52
55
|
],
|
|
53
56
|
"type": "module",
|
|
54
57
|
"sideEffects": false,
|
|
@@ -95,6 +98,19 @@
|
|
|
95
98
|
"solid": "./src/views/app-settings.tsx",
|
|
96
99
|
"import": "./dist/views/app-settings.js"
|
|
97
100
|
},
|
|
101
|
+
"./voice": {
|
|
102
|
+
"types": "./dist/voice/voice.d.ts",
|
|
103
|
+
"solid": "./src/voice/voice.ts",
|
|
104
|
+
"import": "./dist/voice/voice.js"
|
|
105
|
+
},
|
|
106
|
+
"./voice-host": {
|
|
107
|
+
"types": "./dist/voice/voice-host.d.ts",
|
|
108
|
+
"import": "./dist/voice/voice-host.js"
|
|
109
|
+
},
|
|
110
|
+
"./voice-build": {
|
|
111
|
+
"types": "./dist/voice/voice-build.d.ts",
|
|
112
|
+
"import": "./dist/voice/voice-build.js"
|
|
113
|
+
},
|
|
98
114
|
"./app-help": {
|
|
99
115
|
"types": "./dist/views/app-help.d.ts",
|
|
100
116
|
"solid": "./src/views/app-help.tsx",
|
|
@@ -425,10 +441,11 @@
|
|
|
425
441
|
"test:generated": "bun run build && bun scripts/test-generated.ts"
|
|
426
442
|
},
|
|
427
443
|
"dependencies": {
|
|
444
|
+
"@adea-ai/voice": "0.4.0",
|
|
428
445
|
"tar": "^7.5.22"
|
|
429
446
|
},
|
|
430
447
|
"devDependencies": {
|
|
431
|
-
"@adea-ai/ui": "0.160.
|
|
448
|
+
"@adea-ai/ui": "0.160.2",
|
|
432
449
|
"@babel/core": "8.1.0",
|
|
433
450
|
"@babel/preset-typescript": "^8.0.7",
|
|
434
451
|
"@happy-dom/global-registrator": "^21.0.4",
|
|
@@ -453,7 +470,7 @@
|
|
|
453
470
|
"vite-plus": "1.5.1"
|
|
454
471
|
},
|
|
455
472
|
"peerDependencies": {
|
|
456
|
-
"@adea-ai/ui": "^0.160.
|
|
473
|
+
"@adea-ai/ui": "^0.160.2",
|
|
457
474
|
"@lucide/solid": "^1.77.0",
|
|
458
475
|
"@solidjs/web": "^2.0.3",
|
|
459
476
|
"solid-js": "^2.0.3"
|
|
@@ -0,0 +1,579 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The renderer's voice client: one interface the shared Voice view, the dictation button and the
|
|
3
|
+
* read-aloud action use, whether the app runs in the desktop shell (the helpers, over the voice
|
|
4
|
+
* services) or in a browser (`@adea-ai/voice/web`). The person's choices live in a per-device
|
|
5
|
+
* settings store; every call sends the current settings, so the shell never caches a stale choice.
|
|
6
|
+
*/
|
|
7
|
+
import {
|
|
8
|
+
VoiceError,
|
|
9
|
+
route,
|
|
10
|
+
type VoiceProviderAvailability,
|
|
11
|
+
type VoiceProviderKind,
|
|
12
|
+
type VoiceRoutingDecision,
|
|
13
|
+
} from '@adea-ai/voice/contract'
|
|
14
|
+
import {
|
|
15
|
+
createVoiceSettingsStore,
|
|
16
|
+
type VoiceSettingsStorage,
|
|
17
|
+
type VoiceSettingsStore,
|
|
18
|
+
} from '@adea-ai/voice/settings'
|
|
19
|
+
import {
|
|
20
|
+
SystemVoiceProvider,
|
|
21
|
+
browserRecognitionLocal,
|
|
22
|
+
browserRecognitionService,
|
|
23
|
+
type PlatformSpeechHandle,
|
|
24
|
+
type TranscriptionSession,
|
|
25
|
+
} from '@adea-ai/voice/web'
|
|
26
|
+
import {
|
|
27
|
+
VOICE_SERVICES,
|
|
28
|
+
type VoiceEvent,
|
|
29
|
+
type VoiceEventBatch,
|
|
30
|
+
type VoiceServiceName,
|
|
31
|
+
type VoiceSpeechVoice,
|
|
32
|
+
} from './voice-protocol'
|
|
33
|
+
|
|
34
|
+
export type VoicePlatformKind = 'desktop' | 'web'
|
|
35
|
+
|
|
36
|
+
export type VoicePermission = Readonly<{
|
|
37
|
+
state: 'granted' | 'denied' | 'prompt' | 'unavailable'
|
|
38
|
+
microphone: string
|
|
39
|
+
speech: string
|
|
40
|
+
}>
|
|
41
|
+
|
|
42
|
+
export type VoiceProviderStatus = Readonly<{
|
|
43
|
+
id: string
|
|
44
|
+
displayName: string
|
|
45
|
+
kind: VoiceProviderKind
|
|
46
|
+
bundled: boolean
|
|
47
|
+
availability: VoiceProviderAvailability
|
|
48
|
+
}>
|
|
49
|
+
|
|
50
|
+
export type VoiceLocalModel = Readonly<{
|
|
51
|
+
path: string
|
|
52
|
+
architecture: string
|
|
53
|
+
variant: string
|
|
54
|
+
languages: readonly string[]
|
|
55
|
+
streaming: boolean
|
|
56
|
+
}>
|
|
57
|
+
|
|
58
|
+
export type VoiceCapabilitiesView = Readonly<{
|
|
59
|
+
platform: VoicePlatformKind
|
|
60
|
+
speechToText: VoiceRoutingDecision
|
|
61
|
+
textToSpeech: VoiceRoutingDecision
|
|
62
|
+
providers: readonly VoiceProviderStatus[]
|
|
63
|
+
permission: VoicePermission
|
|
64
|
+
localModels: readonly VoiceLocalModel[]
|
|
65
|
+
}>
|
|
66
|
+
|
|
67
|
+
export type VoiceInfo = Readonly<{
|
|
68
|
+
id: string
|
|
69
|
+
name: string
|
|
70
|
+
locale: string
|
|
71
|
+
quality: string
|
|
72
|
+
provider: string
|
|
73
|
+
}>
|
|
74
|
+
|
|
75
|
+
export type LocaleAssetView = Readonly<{ locale: string; status: string; canInstall: boolean }>
|
|
76
|
+
|
|
77
|
+
/** What removing a language's model did, and the state the system reports afterwards. */
|
|
78
|
+
export type ReleasedAssetView = Readonly<{
|
|
79
|
+
locale: string
|
|
80
|
+
/** The model's own locale (`es-MX` for `es-419`). */
|
|
81
|
+
modelLocale: string
|
|
82
|
+
/** False when this app held no claim on the model. */
|
|
83
|
+
released: boolean
|
|
84
|
+
/** Usually `notInstalled`; a model shared with system dictation can stay `installed`. */
|
|
85
|
+
status: string
|
|
86
|
+
canInstall: boolean
|
|
87
|
+
}>
|
|
88
|
+
|
|
89
|
+
export type DictationHandle = Readonly<{
|
|
90
|
+
/** Resolves with the final text; rejects with a `VoiceError` (`cancelled`, `no_speech`, …). */
|
|
91
|
+
result: Promise<string>
|
|
92
|
+
end(): Promise<void>
|
|
93
|
+
cancel(): Promise<void>
|
|
94
|
+
}>
|
|
95
|
+
|
|
96
|
+
export type SpeechHandle = Readonly<{
|
|
97
|
+
id: string
|
|
98
|
+
/**
|
|
99
|
+
* Resolves when audio starts, with the voice speaking when the shell reports it; undefined when
|
|
100
|
+
* the utterance ended before it started. Desktop only.
|
|
101
|
+
*/
|
|
102
|
+
started?: Promise<Readonly<{ voice?: VoiceSpeechVoice }> | undefined>
|
|
103
|
+
/** Resolves when the utterance finishes or is cancelled; rejects with a `VoiceError`. */
|
|
104
|
+
done: Promise<'finished' | 'cancelled'>
|
|
105
|
+
cancel(): Promise<void>
|
|
106
|
+
}>
|
|
107
|
+
|
|
108
|
+
export type InstallHandle = Readonly<{ done: Promise<void>; cancel(): Promise<void> }>
|
|
109
|
+
|
|
110
|
+
export interface VoiceClient {
|
|
111
|
+
readonly platform: VoicePlatformKind
|
|
112
|
+
readonly settings: VoiceSettingsStore
|
|
113
|
+
capabilities(locale?: string): Promise<VoiceCapabilitiesView>
|
|
114
|
+
permission(request?: boolean): Promise<VoicePermission>
|
|
115
|
+
dictate(options?: {
|
|
116
|
+
readonly onPartial?: (text: string) => void
|
|
117
|
+
readonly locale?: string
|
|
118
|
+
}): Promise<DictationHandle>
|
|
119
|
+
speak(text: string, options?: { readonly locale?: string }): Promise<SpeechHandle>
|
|
120
|
+
stopSpeaking(): Promise<void>
|
|
121
|
+
voices(locale?: string): Promise<readonly VoiceInfo[]>
|
|
122
|
+
/** Apple's installable dictation languages (desktop macOS); empty elsewhere. */
|
|
123
|
+
locales(): Promise<readonly LocaleAssetView[]>
|
|
124
|
+
installAssets(
|
|
125
|
+
locale: string,
|
|
126
|
+
onProgress?: (fraction: number | null) => void
|
|
127
|
+
): Promise<InstallHandle>
|
|
128
|
+
/**
|
|
129
|
+
* Removes an installed Apple dictation language (desktop macOS): this app releases its claim on
|
|
130
|
+
* the model and the OS may delete it. `unavailable` elsewhere.
|
|
131
|
+
*/
|
|
132
|
+
releaseAssets(locale: string): Promise<ReleasedAssetView>
|
|
133
|
+
/** Folder picker for BYOM model folders, when the host has one. */
|
|
134
|
+
readonly pickFolder?: () => Promise<string | undefined>
|
|
135
|
+
dispose(): void
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** What a desktop shell transport provides: invoke a named service, get its value or a typed error. */
|
|
139
|
+
export type VoiceTransport = Readonly<{
|
|
140
|
+
invoke(name: VoiceServiceName, input: unknown): Promise<unknown>
|
|
141
|
+
}>
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* A renderer client over a desktop shell. `transport.invoke` must reject with an error whose
|
|
145
|
+
* `code` is the shell's typed error code (a framework `ServiceError` result, or Adea's command
|
|
146
|
+
* error); it is normalised into `VoiceError`.
|
|
147
|
+
*/
|
|
148
|
+
export function createDesktopVoiceClient(options: {
|
|
149
|
+
readonly transport: VoiceTransport
|
|
150
|
+
readonly storage?: VoiceSettingsStorage
|
|
151
|
+
readonly pickFolder?: () => Promise<string | undefined>
|
|
152
|
+
}): VoiceClient {
|
|
153
|
+
const settings = createVoiceSettingsStore(options.storage ?? safeLocalStorage())
|
|
154
|
+
const call = async <T>(name: VoiceServiceName, input: unknown): Promise<T> => {
|
|
155
|
+
try {
|
|
156
|
+
return (await options.transport.invoke(name, input)) as T
|
|
157
|
+
} catch (error) {
|
|
158
|
+
throw VoiceError.wrapping(error)
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
const listeners = new Set<(event: VoiceEvent) => void>()
|
|
162
|
+
let polling = false
|
|
163
|
+
const lifecycle = { disposed: false }
|
|
164
|
+
let cursor = 0
|
|
165
|
+
const poll = async () => {
|
|
166
|
+
if (polling || lifecycle.disposed) return
|
|
167
|
+
polling = true
|
|
168
|
+
let failures = 0
|
|
169
|
+
while (!lifecycle.disposed && listeners.size > 0) {
|
|
170
|
+
try {
|
|
171
|
+
const batch = await call<VoiceEventBatch>(VOICE_SERVICES.events, { after: cursor })
|
|
172
|
+
cursor = batch.cursor
|
|
173
|
+
failures = 0
|
|
174
|
+
for (const event of batch.events) for (const listener of listeners) listener(event)
|
|
175
|
+
} catch {
|
|
176
|
+
// The shell restarted or the channel is reconnecting: back off, then pick up from 0.
|
|
177
|
+
failures++
|
|
178
|
+
cursor = 0
|
|
179
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(5000, 250 * 2 ** failures)))
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
polling = false
|
|
183
|
+
}
|
|
184
|
+
const subscribe = (listener: (event: VoiceEvent) => void) => {
|
|
185
|
+
listeners.add(listener)
|
|
186
|
+
void poll()
|
|
187
|
+
return () => listeners.delete(listener)
|
|
188
|
+
}
|
|
189
|
+
const current = () => settings.read().settings
|
|
190
|
+
|
|
191
|
+
return {
|
|
192
|
+
platform: 'desktop',
|
|
193
|
+
settings,
|
|
194
|
+
...(options.pickFolder ? { pickFolder: options.pickFolder } : {}),
|
|
195
|
+
async capabilities(locale) {
|
|
196
|
+
const view = await call<Omit<VoiceCapabilitiesView, 'platform'>>(
|
|
197
|
+
VOICE_SERVICES.capabilities,
|
|
198
|
+
{
|
|
199
|
+
settings: current(),
|
|
200
|
+
...(locale ? { locale } : {}),
|
|
201
|
+
}
|
|
202
|
+
)
|
|
203
|
+
return { ...view, platform: 'desktop' }
|
|
204
|
+
},
|
|
205
|
+
permission: (request = false) => call<VoicePermission>(VOICE_SERVICES.permission, { request }),
|
|
206
|
+
async dictate(dictateOptions = {}) {
|
|
207
|
+
// Subscribe before starting so no partial between start and subscribe is lost.
|
|
208
|
+
let id: string | undefined
|
|
209
|
+
const early: VoiceEvent[] = []
|
|
210
|
+
let settle!: { resolve: (text: string) => void; reject: (error: VoiceError) => void }
|
|
211
|
+
const result = new Promise<string>((resolve, reject) => {
|
|
212
|
+
settle = { resolve, reject }
|
|
213
|
+
})
|
|
214
|
+
result.catch(() => undefined)
|
|
215
|
+
const handle = (event: VoiceEvent) => {
|
|
216
|
+
if (!('id' in event) || !event.type.startsWith('dictation-')) return
|
|
217
|
+
if (id === undefined) {
|
|
218
|
+
early.push(event)
|
|
219
|
+
return
|
|
220
|
+
}
|
|
221
|
+
if (event.id !== id) return
|
|
222
|
+
if (event.type === 'dictation-partial') dictateOptions.onPartial?.(event.text)
|
|
223
|
+
else {
|
|
224
|
+
unsubscribe()
|
|
225
|
+
if (event.type === 'dictation-final') settle.resolve(event.text)
|
|
226
|
+
else if (event.type === 'dictation-cancelled')
|
|
227
|
+
settle.reject(new VoiceError('cancelled', 'Dictation was cancelled.'))
|
|
228
|
+
else if (event.type === 'dictation-failed')
|
|
229
|
+
settle.reject(new VoiceError(event.code, event.message))
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
const unsubscribe = subscribe(handle)
|
|
233
|
+
try {
|
|
234
|
+
const started = await call<{ id: string }>(VOICE_SERVICES.dictateStart, {
|
|
235
|
+
settings: current(),
|
|
236
|
+
...(dictateOptions.locale ? { locale: dictateOptions.locale } : {}),
|
|
237
|
+
})
|
|
238
|
+
id = started.id
|
|
239
|
+
} catch (error) {
|
|
240
|
+
unsubscribe()
|
|
241
|
+
throw error
|
|
242
|
+
}
|
|
243
|
+
for (const event of early.splice(0)) handle(event)
|
|
244
|
+
const started = id
|
|
245
|
+
return {
|
|
246
|
+
result,
|
|
247
|
+
end: () => call<null>(VOICE_SERVICES.dictateEnd, { id: started }).then(() => undefined),
|
|
248
|
+
cancel: () =>
|
|
249
|
+
call<null>(VOICE_SERVICES.dictateCancel, { id: started }).then(() => undefined),
|
|
250
|
+
}
|
|
251
|
+
},
|
|
252
|
+
async speak(text, speakOptions = {}) {
|
|
253
|
+
let id: string | undefined
|
|
254
|
+
const early: VoiceEvent[] = []
|
|
255
|
+
let settle!: {
|
|
256
|
+
resolve: (outcome: 'finished' | 'cancelled') => void
|
|
257
|
+
reject: (error: VoiceError) => void
|
|
258
|
+
}
|
|
259
|
+
let markStarted!: (start: Readonly<{ voice?: VoiceSpeechVoice }> | undefined) => void
|
|
260
|
+
const started = new Promise<Readonly<{ voice?: VoiceSpeechVoice }> | undefined>((resolve) => {
|
|
261
|
+
markStarted = resolve
|
|
262
|
+
})
|
|
263
|
+
const done = new Promise<'finished' | 'cancelled'>((resolve, reject) => {
|
|
264
|
+
settle = { resolve, reject }
|
|
265
|
+
})
|
|
266
|
+
done.catch(() => undefined)
|
|
267
|
+
const handle = (event: VoiceEvent) => {
|
|
268
|
+
if (!event.type.startsWith('speech-')) return
|
|
269
|
+
if (id === undefined) {
|
|
270
|
+
early.push(event)
|
|
271
|
+
return
|
|
272
|
+
}
|
|
273
|
+
if (event.id !== id) return
|
|
274
|
+
if (event.type === 'speech-started') {
|
|
275
|
+
markStarted(event.voice ? { voice: event.voice } : {})
|
|
276
|
+
} else if (event.type === 'speech-finished' || event.type === 'speech-cancelled') {
|
|
277
|
+
unsubscribe()
|
|
278
|
+
markStarted(undefined)
|
|
279
|
+
settle.resolve(event.type === 'speech-finished' ? 'finished' : 'cancelled')
|
|
280
|
+
} else if (event.type === 'speech-failed') {
|
|
281
|
+
unsubscribe()
|
|
282
|
+
markStarted(undefined)
|
|
283
|
+
settle.reject(new VoiceError(event.code, event.message))
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
const unsubscribe = subscribe(handle)
|
|
287
|
+
try {
|
|
288
|
+
id = (
|
|
289
|
+
await call<{ id: string }>(VOICE_SERVICES.speak, {
|
|
290
|
+
settings: current(),
|
|
291
|
+
text,
|
|
292
|
+
...(speakOptions.locale ? { locale: speakOptions.locale } : {}),
|
|
293
|
+
})
|
|
294
|
+
).id
|
|
295
|
+
} catch (error) {
|
|
296
|
+
unsubscribe()
|
|
297
|
+
throw error
|
|
298
|
+
}
|
|
299
|
+
for (const event of early.splice(0)) handle(event)
|
|
300
|
+
return {
|
|
301
|
+
id,
|
|
302
|
+
started,
|
|
303
|
+
done,
|
|
304
|
+
cancel: () => call<null>(VOICE_SERVICES.speakStop, null).then(() => undefined),
|
|
305
|
+
}
|
|
306
|
+
},
|
|
307
|
+
stopSpeaking: () => call<null>(VOICE_SERVICES.speakStop, null).then(() => undefined),
|
|
308
|
+
voices: async (locale) =>
|
|
309
|
+
(
|
|
310
|
+
await call<{ voices: VoiceInfo[] }>(VOICE_SERVICES.voices, {
|
|
311
|
+
settings: current(),
|
|
312
|
+
...(locale ? { locale } : {}),
|
|
313
|
+
})
|
|
314
|
+
).voices,
|
|
315
|
+
locales: async () =>
|
|
316
|
+
(await call<{ locales: LocaleAssetView[] }>(VOICE_SERVICES.locales, null)).locales,
|
|
317
|
+
async installAssets(locale, onProgress) {
|
|
318
|
+
let id: string | undefined
|
|
319
|
+
const early: VoiceEvent[] = []
|
|
320
|
+
let settle!: { resolve: () => void; reject: (error: VoiceError) => void }
|
|
321
|
+
const done = new Promise<void>((resolve, reject) => {
|
|
322
|
+
settle = { resolve, reject }
|
|
323
|
+
})
|
|
324
|
+
done.catch(() => undefined)
|
|
325
|
+
const handle = (event: VoiceEvent) => {
|
|
326
|
+
if (!event.type.startsWith('install-')) return
|
|
327
|
+
if (id === undefined) {
|
|
328
|
+
early.push(event)
|
|
329
|
+
return
|
|
330
|
+
}
|
|
331
|
+
if (event.id !== id) return
|
|
332
|
+
if (event.type === 'install-progress') onProgress?.(event.fraction)
|
|
333
|
+
else {
|
|
334
|
+
unsubscribe()
|
|
335
|
+
if (event.type === 'install-finished') settle.resolve()
|
|
336
|
+
else if (event.type === 'install-failed')
|
|
337
|
+
settle.reject(new VoiceError(event.code, event.message))
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
const unsubscribe = subscribe(handle)
|
|
341
|
+
try {
|
|
342
|
+
id = (await call<{ id: string }>(VOICE_SERVICES.installAssets, { locale })).id
|
|
343
|
+
} catch (error) {
|
|
344
|
+
unsubscribe()
|
|
345
|
+
throw error
|
|
346
|
+
}
|
|
347
|
+
for (const event of early.splice(0)) handle(event)
|
|
348
|
+
const started = id
|
|
349
|
+
return {
|
|
350
|
+
done,
|
|
351
|
+
cancel: () =>
|
|
352
|
+
call<null>(VOICE_SERVICES.installCancel, { id: started }).then(() => undefined),
|
|
353
|
+
}
|
|
354
|
+
},
|
|
355
|
+
releaseAssets: (locale) => call<ReleasedAssetView>(VOICE_SERVICES.releaseAssets, { locale }),
|
|
356
|
+
dispose() {
|
|
357
|
+
lifecycle.disposed = true
|
|
358
|
+
listeners.clear()
|
|
359
|
+
},
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/**
|
|
364
|
+
* A renderer client for a browser build: on-device browser dictation (the network service only
|
|
365
|
+
* with the person's consent and never in offline-only mode) and platform voices for read-aloud.
|
|
366
|
+
*/
|
|
367
|
+
export function createWebVoiceClient(
|
|
368
|
+
options: { readonly storage?: VoiceSettingsStorage } = {}
|
|
369
|
+
): VoiceClient {
|
|
370
|
+
const settings = createVoiceSettingsStore(options.storage ?? safeLocalStorage())
|
|
371
|
+
const current = () => settings.read().settings
|
|
372
|
+
const local = browserRecognitionLocal()
|
|
373
|
+
const service = browserRecognitionService()
|
|
374
|
+
const voices = new SystemVoiceProvider()
|
|
375
|
+
const allowNetwork = () => !current().privacy.offlineOnly && current().stt.networkConsent
|
|
376
|
+
|
|
377
|
+
const decide = async (locale?: string) => {
|
|
378
|
+
const choice = current()
|
|
379
|
+
const sttLocale = locale ?? choice.stt.locale ?? browserLocale()
|
|
380
|
+
const ttsLocale = choice.tts.locale ?? sttLocale
|
|
381
|
+
const sttProviders = allowNetwork() ? [local, service] : [local]
|
|
382
|
+
const [speechToText, textToSpeech] = await Promise.all([
|
|
383
|
+
route({
|
|
384
|
+
kind: 'speechToText',
|
|
385
|
+
locale: sttLocale,
|
|
386
|
+
providers: sttProviders,
|
|
387
|
+
policy: {
|
|
388
|
+
chains: {
|
|
389
|
+
speechToText: allowNetwork()
|
|
390
|
+
? [local.descriptor.id, service.descriptor.id]
|
|
391
|
+
: [local.descriptor.id],
|
|
392
|
+
},
|
|
393
|
+
...(choice.stt.provider ? { overrides: { speechToText: choice.stt.provider } } : {}),
|
|
394
|
+
allowsOptInModelInstallation: false,
|
|
395
|
+
},
|
|
396
|
+
}),
|
|
397
|
+
route({
|
|
398
|
+
kind: 'textToSpeech',
|
|
399
|
+
locale: ttsLocale,
|
|
400
|
+
providers: [voices],
|
|
401
|
+
policy: {
|
|
402
|
+
chains: { textToSpeech: [voices.descriptor.id] },
|
|
403
|
+
allowsOptInModelInstallation: false,
|
|
404
|
+
},
|
|
405
|
+
}),
|
|
406
|
+
])
|
|
407
|
+
return { speechToText, textToSpeech, sttLocale, ttsLocale }
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
let active: TranscriptionSession | undefined
|
|
411
|
+
const speaking = new Set<PlatformSpeechHandle>()
|
|
412
|
+
return {
|
|
413
|
+
platform: 'web',
|
|
414
|
+
settings,
|
|
415
|
+
async capabilities(locale) {
|
|
416
|
+
const { speechToText, textToSpeech, sttLocale, ttsLocale } = await decide(locale)
|
|
417
|
+
const providers: VoiceProviderStatus[] = await Promise.all(
|
|
418
|
+
[local, service, voices].map(async (provider) => ({
|
|
419
|
+
id: provider.descriptor.id,
|
|
420
|
+
displayName: provider.descriptor.displayName,
|
|
421
|
+
kind: provider.descriptor.kind,
|
|
422
|
+
bundled: true,
|
|
423
|
+
availability: await provider.availability(
|
|
424
|
+
provider.descriptor.kind === 'speechToText' ? sttLocale : ttsLocale
|
|
425
|
+
),
|
|
426
|
+
}))
|
|
427
|
+
)
|
|
428
|
+
return {
|
|
429
|
+
platform: 'web',
|
|
430
|
+
speechToText,
|
|
431
|
+
textToSpeech,
|
|
432
|
+
providers,
|
|
433
|
+
permission: await (async () => {
|
|
434
|
+
const state = await browserMicrophonePermission(false)
|
|
435
|
+
return { state, microphone: state, speech: state }
|
|
436
|
+
})(),
|
|
437
|
+
localModels: [],
|
|
438
|
+
}
|
|
439
|
+
},
|
|
440
|
+
permission: async (request = false) => {
|
|
441
|
+
const state = await browserMicrophonePermission(request)
|
|
442
|
+
return { state, microphone: state, speech: state }
|
|
443
|
+
},
|
|
444
|
+
async dictate(dictateOptions = {}) {
|
|
445
|
+
if (active) throw new VoiceError('busy', 'A dictation is already running.')
|
|
446
|
+
const { speechToText, sttLocale } = await decide(dictateOptions.locale)
|
|
447
|
+
// Routing never selects a network provider on its own. With the person's consent (and not
|
|
448
|
+
// offline-only), the browser's speech service serves a language the device cannot.
|
|
449
|
+
let chosen = speechToText.availability.status === 'available' ? local : undefined
|
|
450
|
+
if (
|
|
451
|
+
!chosen &&
|
|
452
|
+
allowNetwork() &&
|
|
453
|
+
!current().stt.provider?.startsWith('web.speech-recognition-local')
|
|
454
|
+
) {
|
|
455
|
+
const network = await service.availability(sttLocale)
|
|
456
|
+
if (network.status === 'available' || network.unavailable.reason === 'networkRequired')
|
|
457
|
+
chosen = service
|
|
458
|
+
}
|
|
459
|
+
if (!chosen) {
|
|
460
|
+
throw new VoiceError(
|
|
461
|
+
'unavailable',
|
|
462
|
+
speechToText.availability.status === 'unavailable'
|
|
463
|
+
? speechToText.availability.unavailable.message
|
|
464
|
+
: 'Dictation is unavailable in this browser.'
|
|
465
|
+
)
|
|
466
|
+
}
|
|
467
|
+
const session = await chosen.createSession({
|
|
468
|
+
locale: sttLocale,
|
|
469
|
+
allowNetwork: allowNetwork(),
|
|
470
|
+
})
|
|
471
|
+
active = session
|
|
472
|
+
const result = (async () => {
|
|
473
|
+
try {
|
|
474
|
+
for await (const event of session.events()) {
|
|
475
|
+
if (event.type === 'partial')
|
|
476
|
+
dictateOptions.onPartial?.(
|
|
477
|
+
[event.partial.finalizedText, event.partial.volatileText].filter(Boolean).join(' ')
|
|
478
|
+
)
|
|
479
|
+
else if (event.type === 'final') return event.text
|
|
480
|
+
else if (event.type === 'failed') throw event.error
|
|
481
|
+
else if (event.type === 'cancelled')
|
|
482
|
+
throw new VoiceError('cancelled', 'Dictation was cancelled.')
|
|
483
|
+
}
|
|
484
|
+
throw new VoiceError('failed', 'Dictation ended without a result.')
|
|
485
|
+
} finally {
|
|
486
|
+
if (active === session) active = undefined
|
|
487
|
+
}
|
|
488
|
+
})()
|
|
489
|
+
result.catch(() => undefined)
|
|
490
|
+
return { result, end: () => session.end(), cancel: () => session.cancel() }
|
|
491
|
+
},
|
|
492
|
+
async speak(text, speakOptions = {}) {
|
|
493
|
+
const choice = current()
|
|
494
|
+
const locale = speakOptions.locale ?? choice.tts.locale ?? browserLocale()
|
|
495
|
+
const handle = voices.speak(text, {
|
|
496
|
+
locale,
|
|
497
|
+
...(choice.tts.voice ? { voiceName: choice.tts.voice } : {}),
|
|
498
|
+
...(choice.tts.rate ? { rate: choice.tts.rate } : {}),
|
|
499
|
+
allowNetwork: !choice.privacy.offlineOnly && choice.tts.networkConsent,
|
|
500
|
+
})
|
|
501
|
+
speaking.add(handle)
|
|
502
|
+
const done = handle.done
|
|
503
|
+
.finally(() => speaking.delete(handle))
|
|
504
|
+
.then(
|
|
505
|
+
() => 'finished' as const,
|
|
506
|
+
(error: unknown) => {
|
|
507
|
+
const typed = VoiceError.wrapping(error)
|
|
508
|
+
if (typed.code === 'cancelled') return 'cancelled' as const
|
|
509
|
+
throw typed
|
|
510
|
+
}
|
|
511
|
+
)
|
|
512
|
+
done.catch(() => undefined)
|
|
513
|
+
return { id: `web-${Date.now().toString(36)}`, done, cancel: async () => handle.cancel() }
|
|
514
|
+
},
|
|
515
|
+
async stopSpeaking() {
|
|
516
|
+
for (const handle of speaking) handle.cancel()
|
|
517
|
+
},
|
|
518
|
+
async voices(locale) {
|
|
519
|
+
const list = voices.voices(locale ?? current().tts.locale ?? browserLocale())
|
|
520
|
+
return list.local.map((voice) => ({
|
|
521
|
+
id: voice.name,
|
|
522
|
+
name: voice.name,
|
|
523
|
+
locale: voice.lang,
|
|
524
|
+
quality: 'default',
|
|
525
|
+
provider: voices.descriptor.id,
|
|
526
|
+
}))
|
|
527
|
+
},
|
|
528
|
+
locales: async () => [],
|
|
529
|
+
async installAssets() {
|
|
530
|
+
throw new VoiceError('unavailable', 'Language downloads are managed by the browser.')
|
|
531
|
+
},
|
|
532
|
+
async releaseAssets() {
|
|
533
|
+
throw new VoiceError('unavailable', 'Language downloads are managed by the browser.')
|
|
534
|
+
},
|
|
535
|
+
dispose() {
|
|
536
|
+
void active?.cancel()
|
|
537
|
+
},
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/**
|
|
542
|
+
* The microphone permission as the browser reports it. `request` opens the browser prompt (a
|
|
543
|
+
* one-off `getUserMedia` whose tracks stop at once); without it nothing is asked.
|
|
544
|
+
*/
|
|
545
|
+
async function browserMicrophonePermission(
|
|
546
|
+
request: boolean
|
|
547
|
+
): Promise<'granted' | 'denied' | 'prompt' | 'unavailable'> {
|
|
548
|
+
if (typeof navigator === 'undefined') return 'unavailable'
|
|
549
|
+
if (request && navigator.mediaDevices?.getUserMedia) {
|
|
550
|
+
try {
|
|
551
|
+
const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
|
552
|
+
for (const track of stream.getTracks()) track.stop()
|
|
553
|
+
return 'granted'
|
|
554
|
+
} catch (error) {
|
|
555
|
+
return error instanceof DOMException && error.name === 'NotFoundError'
|
|
556
|
+
? 'unavailable'
|
|
557
|
+
: 'denied'
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
try {
|
|
561
|
+
const status = await navigator.permissions?.query({ name: 'microphone' as PermissionName })
|
|
562
|
+
if (status?.state === 'granted' || status?.state === 'denied') return status.state
|
|
563
|
+
return 'prompt'
|
|
564
|
+
} catch {
|
|
565
|
+
return 'prompt'
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
function browserLocale(): string {
|
|
570
|
+
return (typeof navigator === 'undefined' ? undefined : navigator.language) || 'en-US'
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
function safeLocalStorage(): VoiceSettingsStorage | undefined {
|
|
574
|
+
try {
|
|
575
|
+
return typeof localStorage === 'undefined' ? undefined : localStorage
|
|
576
|
+
} catch {
|
|
577
|
+
return undefined
|
|
578
|
+
}
|
|
579
|
+
}
|