arcane-os 0.46.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +1 -1
- package/browser-runtime/ai/browser-speech-recognition.mjs +495 -0
- package/browser-runtime/ai/browser-speech.mjs +1 -0
- package/docs/reference/ai/browser-speech.md +80 -2
- package/docs/reference/availability-and-normalization.md +2 -1
- package/docs/reference/inventory/package-api.json +14 -0
- package/docs/reference/runtime-components.md +23 -0
- package/docs/reference/runtime-modules.md +30 -2
- package/docs/reference/sdk-api.md +57 -0
- package/package.json +1 -1
- package/runtime/arcane/components/speech.html +141 -11
- package/runtime/arcane/components/voice-transcription.html +77 -8
- package/runtime/arcane/modules/AI.js +15 -4
- package/runtime/arcane/modules/AIProviderRuntime.js +118 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.47.0
|
|
4
|
+
|
|
5
|
+
- Add explicitly selected native browser speech recognition through
|
|
6
|
+
`createBrowserSpeechRecognitionProvider` in `arcane-os/ai/browser-speech`.
|
|
7
|
+
The provider uses the browser's SpeechRecognition capability without an SDK
|
|
8
|
+
model download or storage. It declares non-local behavior because the browser
|
|
9
|
+
may use its own remote recognition service. Applications own Profile selection
|
|
10
|
+
and language; Whisper, TTS, typing and existing defaults remain unchanged.
|
|
11
|
+
- Own live capture through the existing independent STT runtime. Start stays in
|
|
12
|
+
the initiating user gesture; Stop drains final recognition, while cancellation,
|
|
13
|
+
provider replacement and cleanup suppress late results. Unsupported browsers
|
|
14
|
+
retain provider construction/catalog and report unavailability on activation.
|
|
15
|
+
- Display native interim recognition immediately in transient shared controls.
|
|
16
|
+
Chat receives one complete combined final transcription at capture end;
|
|
17
|
+
continuous voice appends and saves confirmed segments in order through its
|
|
18
|
+
existing retryable save queue. Interim hypotheses are neither saved nor
|
|
19
|
+
submitted, and existing saved data is unchanged.
|
|
20
|
+
|
|
3
21
|
## 0.46.0
|
|
4
22
|
|
|
5
23
|
- Add public `decodeMarkdownMediaRecord` for complete stored image records or
|
package/README.md
CHANGED
|
@@ -19,7 +19,7 @@ version-locked SDK runtime, while an integrated Arcane checkout uses its live
|
|
|
19
19
|
`arcane/` runtime. Both profiles share the theme, packaging, event, cancellation,
|
|
20
20
|
and browser run contracts while retaining their selected application layout.
|
|
21
21
|
|
|
22
|
-
This checkout defines the `0.
|
|
22
|
+
This checkout defines the `0.47.0` SDK contract. Applications pin one exact npm
|
|
23
23
|
version and lockfile; registry state is deliberately not baked into application
|
|
24
24
|
artifacts.
|
|
25
25
|
|
|
@@ -0,0 +1,495 @@
|
|
|
1
|
+
import Is from '../dependencies/strong-type/index.js';
|
|
2
|
+
import {arcaneLogging} from '../logging.mjs';
|
|
3
|
+
|
|
4
|
+
const is = new Is(false);
|
|
5
|
+
|
|
6
|
+
function recognitionError(message, code, cause) {
|
|
7
|
+
const error = cause === undefined
|
|
8
|
+
? new Error(message)
|
|
9
|
+
: new Error(
|
|
10
|
+
message,
|
|
11
|
+
{cause}
|
|
12
|
+
);
|
|
13
|
+
error.code = code;
|
|
14
|
+
return error;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function recognitionConstructor() {
|
|
18
|
+
if (is.function(globalThis.SpeechRecognition)) return globalThis.SpeechRecognition;
|
|
19
|
+
if (is.function(globalThis.webkitSpeechRecognition)) return globalThis.webkitSpeechRecognition;
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function unavailableRecognition() {
|
|
24
|
+
return recognitionError(
|
|
25
|
+
'Native browser speech recognition is unavailable in this browser.',
|
|
26
|
+
'ARCANE_AI_SPEECH_RECOGNITION_UNAVAILABLE'
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function assertIdentifier(value, label) {
|
|
31
|
+
if (!is.string(value) || !value || value.trim() !== value) {
|
|
32
|
+
throw new TypeError(`${label} must be a nonempty string without surrounding whitespace.`);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Browser-owned live recognition, which may use a remote recognition service.
|
|
38
|
+
* Loading selects the capability; only capture.start() opens a native session.
|
|
39
|
+
*/
|
|
40
|
+
export function createBrowserSpeechRecognitionProvider(
|
|
41
|
+
{id, model, language} = {}
|
|
42
|
+
) {
|
|
43
|
+
assertIdentifier(id, 'Browser speech recognition provider id');
|
|
44
|
+
assertIdentifier(model?.id, 'Browser speech recognition model id');
|
|
45
|
+
if (language !== undefined && !is.string(language)) {
|
|
46
|
+
throw new TypeError('Browser speech recognition language must be a string.');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const modelId = model.id;
|
|
50
|
+
let state = 'unloaded';
|
|
51
|
+
let generation = 0;
|
|
52
|
+
let activeCapture = null;
|
|
53
|
+
let unloadOperation = null;
|
|
54
|
+
let disposeOperation = null;
|
|
55
|
+
|
|
56
|
+
function authority() {
|
|
57
|
+
return {
|
|
58
|
+
protocol: 'arcane-ai-model-authority/1',
|
|
59
|
+
providerId: id,
|
|
60
|
+
modelId,
|
|
61
|
+
role: 'stt',
|
|
62
|
+
localOnly: false
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function assertSelection(selection) {
|
|
67
|
+
if (selection && (selection.providerId !== id
|
|
68
|
+
|| selection.modelId !== modelId
|
|
69
|
+
|| selection.localOnly === true
|
|
70
|
+
|| (selection.role !== undefined && selection.role !== 'stt'))) {
|
|
71
|
+
throw recognitionError(
|
|
72
|
+
'The selected browser speech model does not match this provider.',
|
|
73
|
+
'ARCANE_AI_MODEL_AUTHORITY_REQUIRED'
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function assertContext(context) {
|
|
79
|
+
if (context.role !== undefined && context.role !== 'stt') {
|
|
80
|
+
throw recognitionError('This provider supports only STT.', 'ARCANE_AI_INVALID_REQUEST');
|
|
81
|
+
}
|
|
82
|
+
assertSelection(context.selection);
|
|
83
|
+
if (context.signal?.aborted) {
|
|
84
|
+
const error = recognitionError(
|
|
85
|
+
'The browser speech operation was cancelled.',
|
|
86
|
+
'ARCANE_AI_REQUEST_ABORTED',
|
|
87
|
+
context.signal.reason
|
|
88
|
+
);
|
|
89
|
+
error.name = 'AbortError';
|
|
90
|
+
throw error;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function status() {
|
|
95
|
+
const loaded = state === 'ready';
|
|
96
|
+
return {
|
|
97
|
+
providerId: id,
|
|
98
|
+
role: 'stt',
|
|
99
|
+
modelId,
|
|
100
|
+
localOnly: false,
|
|
101
|
+
state,
|
|
102
|
+
loaded,
|
|
103
|
+
busy: activeCapture !== null,
|
|
104
|
+
execution: {
|
|
105
|
+
requestedDevice: 'browser-native',
|
|
106
|
+
selectedDevice: loaded ? 'browser-native' : null,
|
|
107
|
+
maxConcurrentRequests: 1,
|
|
108
|
+
activeRequestCount: activeCapture === null ? 0 : 1
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function createCapture(
|
|
114
|
+
{
|
|
115
|
+
onSegment,
|
|
116
|
+
onInterim,
|
|
117
|
+
onState,
|
|
118
|
+
onError,
|
|
119
|
+
language: captureLanguage = language,
|
|
120
|
+
continuous = true
|
|
121
|
+
} = {}
|
|
122
|
+
) {
|
|
123
|
+
const captureGeneration = generation;
|
|
124
|
+
let recognizer = null;
|
|
125
|
+
let signal = null;
|
|
126
|
+
let started = false;
|
|
127
|
+
let nativeStarted = false;
|
|
128
|
+
let receivedStart = false;
|
|
129
|
+
let stopping = false;
|
|
130
|
+
let cancelled = false;
|
|
131
|
+
let failed = false;
|
|
132
|
+
let settled = false;
|
|
133
|
+
let captureState = null;
|
|
134
|
+
let lastFinalIndex = -1;
|
|
135
|
+
let sequence = 0;
|
|
136
|
+
let resolveStarted = null;
|
|
137
|
+
let resolveDone;
|
|
138
|
+
const done = new Promise(
|
|
139
|
+
function waitForRecognitionEnd(resolve) {
|
|
140
|
+
resolveDone = resolve;
|
|
141
|
+
}
|
|
142
|
+
);
|
|
143
|
+
|
|
144
|
+
function reportError(error) {
|
|
145
|
+
if (!is.function(onError)) {
|
|
146
|
+
arcaneLogging.error('Browser speech recognition failed:', error);
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
try {
|
|
150
|
+
const operation = onError(error);
|
|
151
|
+
Promise.resolve(operation).catch(reportErrorCallbackFailure);
|
|
152
|
+
} catch (callbackError) {
|
|
153
|
+
reportErrorCallbackFailure(callbackError);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function reportErrorCallbackFailure(error) {
|
|
158
|
+
arcaneLogging.error('Browser speech recognition error handler failed:', error);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function observeCallbackFailure(error) {
|
|
162
|
+
const callbackError = recognitionError(
|
|
163
|
+
'A browser speech recognition callback failed.',
|
|
164
|
+
'ARCANE_AI_SPEECH_CALLBACK_FAILED',
|
|
165
|
+
error
|
|
166
|
+
);
|
|
167
|
+
if (settled || cancelled || failed) {
|
|
168
|
+
reportError(callbackError);
|
|
169
|
+
return;
|
|
170
|
+
}
|
|
171
|
+
interrupt(callbackError);
|
|
172
|
+
abortNative();
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function notify(callback, value) {
|
|
176
|
+
if (!is.function(callback)) return;
|
|
177
|
+
try {
|
|
178
|
+
const operation = callback(value);
|
|
179
|
+
Promise.resolve(operation).catch(observeCallbackFailure);
|
|
180
|
+
} catch (error) {
|
|
181
|
+
observeCallbackFailure(error);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function publishState(nextState) {
|
|
186
|
+
if (captureState === nextState) return;
|
|
187
|
+
captureState = nextState;
|
|
188
|
+
notify(onState, nextState);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function settleStarted(success) {
|
|
192
|
+
if (!resolveStarted) return;
|
|
193
|
+
const resolve = resolveStarted;
|
|
194
|
+
resolveStarted = null;
|
|
195
|
+
resolve(success);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function finish(success) {
|
|
199
|
+
if (settled) return;
|
|
200
|
+
settled = true;
|
|
201
|
+
nativeStarted = false;
|
|
202
|
+
signal?.removeEventListener('abort', cancel);
|
|
203
|
+
signal = null;
|
|
204
|
+
if (recognizer) {
|
|
205
|
+
recognizer.onstart = null;
|
|
206
|
+
recognizer.onresult = null;
|
|
207
|
+
recognizer.onerror = null;
|
|
208
|
+
recognizer.onend = null;
|
|
209
|
+
recognizer = null;
|
|
210
|
+
}
|
|
211
|
+
if (activeCapture === capture) activeCapture = null;
|
|
212
|
+
settleStarted(false);
|
|
213
|
+
publishState(failed ? 'interrupted' : 'stopped');
|
|
214
|
+
resolveDone(success);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function interrupt(error) {
|
|
218
|
+
if (failed || settled) return;
|
|
219
|
+
failed = true;
|
|
220
|
+
settleStarted(false);
|
|
221
|
+
reportError(error);
|
|
222
|
+
publishState('interrupted');
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function abortNative() {
|
|
226
|
+
if (settled) return;
|
|
227
|
+
if (!nativeStarted) {
|
|
228
|
+
finish(false);
|
|
229
|
+
return;
|
|
230
|
+
}
|
|
231
|
+
try {
|
|
232
|
+
recognizer.abort();
|
|
233
|
+
} catch (error) {
|
|
234
|
+
const abortError = recognitionError(
|
|
235
|
+
'Native browser speech recognition could not be cancelled.',
|
|
236
|
+
'ARCANE_AI_SPEECH_RECOGNITION_ABORT_FAILED',
|
|
237
|
+
error
|
|
238
|
+
);
|
|
239
|
+
if (failed) reportError(abortError);
|
|
240
|
+
else interrupt(abortError);
|
|
241
|
+
// An unsuccessful abort is not evidence of native end. Keep
|
|
242
|
+
// the session owned until the browser reports disconnection.
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function cancel() {
|
|
247
|
+
if (settled || cancelled) return done;
|
|
248
|
+
cancelled = true;
|
|
249
|
+
settleStarted(false);
|
|
250
|
+
abortNative();
|
|
251
|
+
return done;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function recognitionStarted() {
|
|
255
|
+
if (settled || cancelled || failed) return;
|
|
256
|
+
receivedStart = true;
|
|
257
|
+
if (!stopping) publishState('listening');
|
|
258
|
+
settleStarted(!cancelled && !failed && !stopping);
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
function recognitionResults(event) {
|
|
262
|
+
if (settled || cancelled || failed) return;
|
|
263
|
+
try {
|
|
264
|
+
for (let index = event.resultIndex; index < event.results.length; index += 1) {
|
|
265
|
+
if (settled || cancelled || failed) return;
|
|
266
|
+
const result = event.results[index];
|
|
267
|
+
if (!result.isFinal || index <= lastFinalIndex) continue;
|
|
268
|
+
lastFinalIndex = index;
|
|
269
|
+
sequence += 1;
|
|
270
|
+
notify(
|
|
271
|
+
onSegment,
|
|
272
|
+
{text: result[0].transcript, sequence}
|
|
273
|
+
);
|
|
274
|
+
}
|
|
275
|
+
if (settled || cancelled || failed) return;
|
|
276
|
+
let text = '';
|
|
277
|
+
// Native results retain their final prefix and replace only
|
|
278
|
+
// interim entries. Do not revisit that growing final prefix.
|
|
279
|
+
for (let index = lastFinalIndex + 1; index < event.results.length; index += 1) {
|
|
280
|
+
text += event.results[index][0].transcript;
|
|
281
|
+
}
|
|
282
|
+
notify(
|
|
283
|
+
onInterim,
|
|
284
|
+
{text}
|
|
285
|
+
);
|
|
286
|
+
} catch (error) {
|
|
287
|
+
interrupt(
|
|
288
|
+
recognitionError(
|
|
289
|
+
'The browser speech recognition result could not be read.',
|
|
290
|
+
'ARCANE_AI_INVALID_PROVIDER_RESULT',
|
|
291
|
+
error
|
|
292
|
+
)
|
|
293
|
+
);
|
|
294
|
+
abortNative();
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function recognitionFailed(event) {
|
|
299
|
+
if (settled || cancelled || failed) return;
|
|
300
|
+
const error = recognitionError(
|
|
301
|
+
event.message || `Native browser speech recognition failed: ${event.error}.`,
|
|
302
|
+
'ARCANE_AI_SPEECH_RECOGNITION_FAILED',
|
|
303
|
+
event
|
|
304
|
+
);
|
|
305
|
+
error.reason = event.error;
|
|
306
|
+
interrupt(error);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
function recognitionEnded() {
|
|
310
|
+
finish(receivedStart && !cancelled && !failed);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
function start(
|
|
314
|
+
{signal: captureSignal} = {}
|
|
315
|
+
) {
|
|
316
|
+
if (started || settled) return Promise.resolve(false);
|
|
317
|
+
started = true;
|
|
318
|
+
const starting = new Promise(
|
|
319
|
+
function waitForRecognitionStart(resolve) {
|
|
320
|
+
resolveStarted = resolve;
|
|
321
|
+
}
|
|
322
|
+
);
|
|
323
|
+
signal = captureSignal ?? null;
|
|
324
|
+
if (signal?.aborted) {
|
|
325
|
+
cancel();
|
|
326
|
+
return starting;
|
|
327
|
+
}
|
|
328
|
+
try {
|
|
329
|
+
if (state === 'disposed' || disposeOperation) {
|
|
330
|
+
throw recognitionError('The browser speech provider is disposed.', 'ARCANE_AI_PROVIDER_DISPOSED');
|
|
331
|
+
}
|
|
332
|
+
if (state !== 'ready' || unloadOperation || generation !== captureGeneration) {
|
|
333
|
+
throw recognitionError('The browser speech provider is not ready.', 'ARCANE_AI_NOT_READY');
|
|
334
|
+
}
|
|
335
|
+
if (activeCapture !== null) {
|
|
336
|
+
throw recognitionError('The browser speech provider is at capacity.', 'ARCANE_AI_PROVIDER_BUSY');
|
|
337
|
+
}
|
|
338
|
+
const Recognition = recognitionConstructor();
|
|
339
|
+
if (!Recognition) throw unavailableRecognition();
|
|
340
|
+
activeCapture = capture;
|
|
341
|
+
recognizer = new Recognition();
|
|
342
|
+
if (captureLanguage !== undefined) recognizer.lang = captureLanguage;
|
|
343
|
+
recognizer.continuous = continuous;
|
|
344
|
+
recognizer.interimResults = true;
|
|
345
|
+
recognizer.maxAlternatives = 1;
|
|
346
|
+
recognizer.onstart = recognitionStarted;
|
|
347
|
+
recognizer.onresult = recognitionResults;
|
|
348
|
+
recognizer.onerror = recognitionFailed;
|
|
349
|
+
recognizer.onend = recognitionEnded;
|
|
350
|
+
signal?.addEventListener(
|
|
351
|
+
'abort',
|
|
352
|
+
cancel,
|
|
353
|
+
{once: true}
|
|
354
|
+
);
|
|
355
|
+
publishState('starting');
|
|
356
|
+
if (settled || cancelled || failed) return starting;
|
|
357
|
+
nativeStarted = true;
|
|
358
|
+
// This call stays in the initiating user gesture. No model
|
|
359
|
+
// loading, permission preflight, or promise precedes it.
|
|
360
|
+
recognizer.start();
|
|
361
|
+
} catch (error) {
|
|
362
|
+
interrupt(
|
|
363
|
+
recognitionError(
|
|
364
|
+
'Native browser speech recognition could not start.',
|
|
365
|
+
is.string(error?.code) ? error.code : 'ARCANE_AI_SPEECH_RECOGNITION_START_FAILED',
|
|
366
|
+
error
|
|
367
|
+
)
|
|
368
|
+
);
|
|
369
|
+
finish(false);
|
|
370
|
+
}
|
|
371
|
+
return starting;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
function stop() {
|
|
375
|
+
if (settled || stopping || cancelled || failed) return done;
|
|
376
|
+
if (!started || !nativeStarted) return cancel();
|
|
377
|
+
stopping = true;
|
|
378
|
+
publishState('stopping');
|
|
379
|
+
if (settled || cancelled || failed) return done;
|
|
380
|
+
try {
|
|
381
|
+
recognizer.stop();
|
|
382
|
+
} catch (error) {
|
|
383
|
+
interrupt(
|
|
384
|
+
recognitionError(
|
|
385
|
+
'Native browser speech recognition could not stop.',
|
|
386
|
+
'ARCANE_AI_SPEECH_RECOGNITION_STOP_FAILED',
|
|
387
|
+
error
|
|
388
|
+
)
|
|
389
|
+
);
|
|
390
|
+
abortNative();
|
|
391
|
+
}
|
|
392
|
+
return done;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const capture = {start, stop, cancel, destroy: cancel, done};
|
|
396
|
+
return capture;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
const provider = {
|
|
400
|
+
protocol: 'arcane-ai-provider/2',
|
|
401
|
+
role: 'stt',
|
|
402
|
+
id,
|
|
403
|
+
localOnly: false,
|
|
404
|
+
maxConcurrentRequests: 1,
|
|
405
|
+
|
|
406
|
+
catalog() {
|
|
407
|
+
return [{id: modelId, providerId: id, role: 'stt', localOnly: false}];
|
|
408
|
+
},
|
|
409
|
+
|
|
410
|
+
async inspect(selection, context = {}) {
|
|
411
|
+
assertContext(context);
|
|
412
|
+
assertSelection(selection);
|
|
413
|
+
if (state === 'disposed' || disposeOperation) {
|
|
414
|
+
return {
|
|
415
|
+
available: false,
|
|
416
|
+
code: 'ARCANE_AI_PROVIDER_DISPOSED',
|
|
417
|
+
message: 'The browser speech provider is disposed.'
|
|
418
|
+
};
|
|
419
|
+
}
|
|
420
|
+
if (!recognitionConstructor()) {
|
|
421
|
+
const error = unavailableRecognition();
|
|
422
|
+
return {available: false, code: error.code, message: error.message};
|
|
423
|
+
}
|
|
424
|
+
return {available: true, authority: authority()};
|
|
425
|
+
},
|
|
426
|
+
|
|
427
|
+
status,
|
|
428
|
+
createCapture,
|
|
429
|
+
|
|
430
|
+
async load(context = {}) {
|
|
431
|
+
assertContext(context);
|
|
432
|
+
if (state === 'disposed' || disposeOperation) {
|
|
433
|
+
throw recognitionError('The browser speech provider is disposed.', 'ARCANE_AI_PROVIDER_DISPOSED');
|
|
434
|
+
}
|
|
435
|
+
if (unloadOperation) {
|
|
436
|
+
throw recognitionError('The browser speech provider is unloading.', 'ARCANE_AI_OPERATION_SUPERSEDED');
|
|
437
|
+
}
|
|
438
|
+
if (!recognitionConstructor()) {
|
|
439
|
+
state = 'error';
|
|
440
|
+
throw unavailableRecognition();
|
|
441
|
+
}
|
|
442
|
+
state = 'ready';
|
|
443
|
+
return status();
|
|
444
|
+
},
|
|
445
|
+
|
|
446
|
+
async request(context = {}) {
|
|
447
|
+
assertContext(context);
|
|
448
|
+
throw recognitionError(
|
|
449
|
+
'Native browser speech recognition accepts live microphone capture only; arbitrary audio files cannot be transcribed.',
|
|
450
|
+
'ARCANE_AI_SPEECH_INPUT_UNSUPPORTED'
|
|
451
|
+
);
|
|
452
|
+
},
|
|
453
|
+
|
|
454
|
+
unload(context = {}) {
|
|
455
|
+
assertContext(context);
|
|
456
|
+
if (state === 'disposed') {
|
|
457
|
+
return Promise.resolve(
|
|
458
|
+
status()
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
if (unloadOperation) return unloadOperation;
|
|
462
|
+
state = 'unloading';
|
|
463
|
+
generation += 1;
|
|
464
|
+
const capture = activeCapture;
|
|
465
|
+
unloadOperation = Promise.resolve(capture?.done).then(
|
|
466
|
+
function finishRecognitionUnload() {
|
|
467
|
+
state = 'unloaded';
|
|
468
|
+
unloadOperation = null;
|
|
469
|
+
return status();
|
|
470
|
+
}
|
|
471
|
+
);
|
|
472
|
+
capture?.cancel();
|
|
473
|
+
return unloadOperation;
|
|
474
|
+
},
|
|
475
|
+
|
|
476
|
+
dispose(context = {}) {
|
|
477
|
+
assertContext(context);
|
|
478
|
+
if (state === 'disposed') {
|
|
479
|
+
return Promise.resolve(
|
|
480
|
+
status()
|
|
481
|
+
);
|
|
482
|
+
}
|
|
483
|
+
if (disposeOperation) return disposeOperation;
|
|
484
|
+
disposeOperation = provider.unload(context).then(
|
|
485
|
+
function finishRecognitionDispose() {
|
|
486
|
+
state = 'disposed';
|
|
487
|
+
disposeOperation = null;
|
|
488
|
+
return status();
|
|
489
|
+
}
|
|
490
|
+
);
|
|
491
|
+
return disposeOperation;
|
|
492
|
+
}
|
|
493
|
+
};
|
|
494
|
+
return provider;
|
|
495
|
+
}
|
|
@@ -10,6 +10,7 @@ export {
|
|
|
10
10
|
createBrowserKokoroProvider,
|
|
11
11
|
createBrowserWhisperProvider,
|
|
12
12
|
} from "./browser-speech-providers.mjs";
|
|
13
|
+
export {createBrowserSpeechRecognitionProvider} from './browser-speech-recognition.mjs';
|
|
13
14
|
export {
|
|
14
15
|
createTwinCloudTTSProvider,
|
|
15
16
|
createTwinCloudTTSProvider as createDigitalOceanFalTTSProvider,
|
|
@@ -1,11 +1,87 @@
|
|
|
1
1
|
# Browser speech providers
|
|
2
2
|
|
|
3
|
-
`arcane-os/ai/browser-speech` supplies caller-selected Whisper
|
|
4
|
-
Kokoro text-to-speech, and TWiN Cloud text-to-speech providers. Its local path
|
|
3
|
+
`arcane-os/ai/browser-speech` supplies caller-selected Whisper or native Web
|
|
4
|
+
Speech recognition, Kokoro text-to-speech, and TWiN Cloud text-to-speech providers. Its local path
|
|
5
5
|
provides artifact storage, live module routing, role Workers, provider/2
|
|
6
6
|
adapters, bounded parallel TTS synthesis, audio normalization, cancellation,
|
|
7
7
|
and cleanup.
|
|
8
8
|
|
|
9
|
+
## Native browser speech recognition
|
|
10
|
+
|
|
11
|
+
Select the browser's `SpeechRecognition` or `webkitSpeechRecognition` service
|
|
12
|
+
explicitly. It may use a remote browser service, so this provider declares
|
|
13
|
+
`localOnly:false`; it is never substituted for Whisper automatically. The
|
|
14
|
+
application owns the provider/model identifiers and saved Profile selection.
|
|
15
|
+
|
|
16
|
+
```javascript
|
|
17
|
+
import {createBrowserSpeechRecognitionProvider} from 'arcane-os/ai/browser-speech';
|
|
18
|
+
|
|
19
|
+
async function selectBrowserRecognition(ai, signal) {
|
|
20
|
+
const provider = createBrowserSpeechRecognitionProvider(
|
|
21
|
+
{id: 'WEB_SPEECH', model: {id: 'WEB_SPEECH'}}
|
|
22
|
+
);
|
|
23
|
+
await ai.configureSpeechProvider('stt', provider, {modelId: 'WEB_SPEECH', signal});
|
|
24
|
+
}
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Configuration and activation use the existing independent STT lifecycle.
|
|
28
|
+
Construction and `catalog()` succeed when the native API is absent;
|
|
29
|
+
`inspect()` reports unavailable and `load()` reports an unsupported error.
|
|
30
|
+
Neither factory, configuration nor load opens a microphone, downloads a
|
|
31
|
+
model, accesses storage, or changes TTS, typing, existing defaults, or saved
|
|
32
|
+
data. Browser/OS availability, microphone permission, language support and
|
|
33
|
+
service connectivity remain native platform requirements across Windows,
|
|
34
|
+
Linux, macOS and Android hosts.
|
|
35
|
+
|
|
36
|
+
After STT activation, the existing `speech.html` and `voice-transcription.html`
|
|
37
|
+
recording controls select live capture automatically. `speech.html` retains
|
|
38
|
+
every final result and emits one complete `speech-transcription-complete`
|
|
39
|
+
message when capture ends. Its live display retains finalized words alongside
|
|
40
|
+
the current interim text while speaking. The voice component appends each final result
|
|
41
|
+
through its existing ordered save queue. Interim text appears only in a
|
|
42
|
+
transient surface and never enters a save or completion event. A custom voice
|
|
43
|
+
component `transcribe(file,context)` callback retains the existing clip path.
|
|
44
|
+
Final segments waiting behind an application save remain visible in that
|
|
45
|
+
transient surface until the ordered queue appends them to the main transcript.
|
|
46
|
+
|
|
47
|
+
For direct use, `ai.supportsTranscriptionCapture()` detects the selected
|
|
48
|
+
provider's capture capability. `ai.createTranscriptionCapture(options)` returns
|
|
49
|
+
`null` for a file-only provider or a single-use capture handle with
|
|
50
|
+
`start({signal})`, `stop()`, `cancel()`, `destroy()` and `done`.
|
|
51
|
+
Options are `{language,continuous,onSegment,onInterim,onState,onError}`;
|
|
52
|
+
`onSegment({text,sequence})` receives each complete final result once, and
|
|
53
|
+
`onInterim({text})` receives the current mutable hypothesis. Callbacks are
|
|
54
|
+
observational and never delay native capture. The voice component, rather than
|
|
55
|
+
the capture callback, owns asynchronous ordered application saves.
|
|
56
|
+
|
|
57
|
+
Call `start()` directly from the initiating user gesture. Its promise resolves
|
|
58
|
+
to whether native capture actually started. `stop()` asks the browser to stop
|
|
59
|
+
listening and finish recognition, retaining final results through native `end`;
|
|
60
|
+
`stop()` and `done` resolve `true` for normal native completion and `false` for
|
|
61
|
+
cancellation or failure after capture settlement. Runtime cleanup failure
|
|
62
|
+
rejects the runtime handle's promises. `cancel()`, an aborted signal,
|
|
63
|
+
provider replacement, unload, and disposal request native `abort()` and suppress
|
|
64
|
+
late text. Capture is an owned busy STT operation; it is never delayed into a
|
|
65
|
+
request queue that would lose the gesture. Natural native `end` stops the
|
|
66
|
+
session without automatic restart. The user can start another session.
|
|
67
|
+
If native `abort()` itself throws, the complete failure remains observable and
|
|
68
|
+
the capture stays owned until native `end`; requesting abort is not evidence
|
|
69
|
+
that the browser microphone has disconnected.
|
|
70
|
+
|
|
71
|
+
Missing API reports `ARCANE_AI_SPEECH_RECOGNITION_UNAVAILABLE`; file input
|
|
72
|
+
reports `ARCANE_AI_SPEECH_INPUT_UNSUPPORTED`. Native service errors use
|
|
73
|
+
`ARCANE_AI_SPEECH_RECOGNITION_FAILED`, retain the browser reason and complete
|
|
74
|
+
event as their cause, and remain available in developer diagnostics. Shared
|
|
75
|
+
ordinary controls show concise failure status. Confirmed final words received
|
|
76
|
+
before a service failure remain deliverable; explicit cancellation or
|
|
77
|
+
supersession suppresses the pending chat submission.
|
|
78
|
+
|
|
79
|
+
This API captures live microphone speech. `AI.fetchSTT(file)` remains the
|
|
80
|
+
Whisper/file-provider contract; native recognition rejects file transcription
|
|
81
|
+
honestly rather than recording again or silently switching providers. See the
|
|
82
|
+
[Web Speech specification](https://webaudio.github.io/web-speech-api/#speechreco-methods)
|
|
83
|
+
for native stop/abort behavior.
|
|
84
|
+
|
|
9
85
|
<a id="digitalocean-fal-text-to-speech"></a>
|
|
10
86
|
|
|
11
87
|
## TWiN Cloud text-to-speech
|
|
@@ -971,6 +1047,7 @@ records; this contract does not freeze them or shorten their content.
|
|
|
971
1047
|
| Native WebView | Conditional | Available when the WebView exposes the browser APIs above. It does not invoke Core speech. |
|
|
972
1048
|
| Node | Remote adapter available; local execution unavailable | TWiN Cloud uses Fetch and Blob. The local providers have no SDK Node speech-storage, Worker, or audio-decoder host. |
|
|
973
1049
|
| Remote TTS | Explicitly selected | TWiN Cloud uses standard Fetch and Blob with an application-owned key; no Worker, OPFS, or local model is needed for this adapter. |
|
|
1050
|
+
| Native browser STT | Explicitly selected, conditional | Requires `SpeechRecognition` or `webkitSpeechRecognition` and native microphone/service availability. No Worker, OPFS, or SDK model download; locality is not promised. |
|
|
974
1051
|
|
|
975
1052
|
STT and TTS own independent provider lifecycles. A failure or cancellation in
|
|
976
1053
|
one role does not disable the other role or authorize a fallback provider.
|
|
@@ -982,6 +1059,7 @@ import {
|
|
|
982
1059
|
BROWSER_SPEECH_ARTIFACT_GRAPH_PROTOCOL,
|
|
983
1060
|
BROWSER_SPEECH_ARTIFACT_PROTOCOL,
|
|
984
1061
|
createBrowserKokoroProvider,
|
|
1062
|
+
createBrowserSpeechRecognitionProvider,
|
|
985
1063
|
createBrowserSpeechArtifactGraph,
|
|
986
1064
|
createBrowserSpeechAuthority,
|
|
987
1065
|
createBrowserWhisperProvider,
|
|
@@ -72,6 +72,7 @@ version; WebKitGTK availability must not be generalized to macOS.
|
|
|
72
72
|
| Select and observe independent LLM/STT/TTS roles | `/arcane/modules/AIProviderRuntime.js` and `AIRuntimeState.js` | **Cross-host** controller/state; registered providers retain their own host requirements | Required/projected provider members, route/configuration records, and status fields; per-role lifecycle, cancellation, stream cleanup, sticky state, and startup barriers are normalized. `localOnly` creates no fallback. |
|
|
73
73
|
| Run a caller-selected local LLM entirely in a browser renderer | `arcane-os/ai/browser-wasm` through `createArcaneAI()` | **Browser** only; secure context, WebAssembly, and OPFS/DBOPFS; full WebGPU offload by default or explicit CPU with `gpuLayers:0` | The public AI API module normalizes multi-model lifecycle, status, complete all-choice streaming, cancellation, exact ordered structural tool-call visibility, and session persistence. Model sources are canonical ordered file descriptors; licenses and model choice remain application policy. |
|
|
74
74
|
| Run caller-selected Whisper or Kokoro in a browser renderer | `arcane-os/ai/browser-speech` registered with `AIProviderRuntime` | **Browser** only; DBOPFS, Web Locks, Workers, Fetch/object URLs, and a caller-supplied self-contained runtime/model closure are required | STT/TTS use independent provider/2 lifecycle and status. Kokoro adds bounded Worker/session concurrency and explicit `auto`, `webgpu`, or `wasm` execution. Complete model/runtime selection, offline behavior, cancellation, Worker teardown, and request/result shapes are normalized. No runtime/model content or cloud fallback is supplied. |
|
|
75
|
+
| Capture speech with the explicitly selected native browser service | `createBrowserSpeechRecognitionProvider` from `arcane-os/ai/browser-speech`, configured as STT | **Browser / native WebView**, conditional on `SpeechRecognition` or `webkitSpeechRecognition`; no SDK model, Worker, or storage required | `localOnly:false` because the browser may use a remote service. Factory/catalog remain usable without native support; inspect/load report availability. User-gesture start, exact final results, transient interim text, stop/drain and abort are owned by the provider/runtime and shared controls. No provider fallback. |
|
|
75
76
|
| Prepare ordered speech playback or reuse the speech-input formatting filter | `arcane-os/speech-playback` and `arcane-os/speech-text` | **Node** with injected media adapters, or **Browser / Native WebView** media; the text filter itself is **Cross-host** | Stored and caller-owned text stays exact. Only the outbound synthesis copy automatically loses repeated same formatting marks. A capacity-advertising provider receives complete segments immediately while retaining indexed playback; native/custom synthesis stays serialized. |
|
|
76
77
|
| Preserve complete chat history and memory | `/arcane/modules/PersistentAIChatSession.js` | **Browser / native WebView** with ChatEntity/DBOPFS and a configured chat function | Existing DBOPFS names and memory semantics are preserved. Live-context commit is atomic; durable persistence is explicit and coherent across user/assistant turns and atomic all-ID tool-result batches. |
|
|
77
78
|
| Search an app-owned document corpus for explicit chat context | `/arcane/modules/DBOPFSDocumentLibrary.js` | **Browser** or compatible injected DBOPFS adapter | Generation/manifest completion, complete lexical search, partial read failures, and untrusted context labels are normalized. Construction does not search; an explicitly wired context builder performs retrieval for each prepared chat send. |
|
|
@@ -79,7 +80,7 @@ version; WebKitGTK availability must not be generalized to macOS.
|
|
|
79
80
|
| Use local AI without coupling app code to Ollama HTTP | `Arcane.localAI`, `Arcane.ai`, or `/arcane/modules/Ollama.js` | Primarily **Native**; Android exposes a narrower admitted inference projection | Admission, errors, and managed-operation events are normalized. Direct Ollama response envelopes remain **Provider-native**. |
|
|
80
81
|
| Use TWiN Cloud from the renderer profile | `/arcane/modules/AI.js` | **Cloud** from an allowed browser/native renderer | High-level chat behavior is normalized by the module. The TWiN access key authenticates remote LLM chat; raw provider diagnostics remain provider-specific. No automatic cloud fallback is inferred from local failure. |
|
|
81
82
|
| Send one TWiN Cloud request with an explicit key and model | `fetchRequest` from `arcane-os/ai/twin-cloud` | **Node** and **Browser**, using standard Fetch and a remote HTTPS provider | Keeps complete messages and returns the full parsed provider JSON. Rejected Fetch and HTTP 529 share three retries after 3000 ms; HTTP 429 overload retains unlimited three-second retries. Optional `onRetry` reports waiting/requesting phases without blocking recovery. Shared structured-output mapping and cancellation match browser TWiN transport. No browser profile, AI/user singleton, DBOPFS, or retained request history is created. |
|
|
82
|
-
| Use speech through one application helper | `/arcane/modules/AI.js` and `Arcane.speech` | **Browser** or **Native** |
|
|
83
|
+
| Use speech through one application helper | `/arcane/modules/AI.js` and `Arcane.speech` | **Browser** or **Native** | Whisper and Kokoro keep their on-device paths; explicitly selected Web Speech STT and TWiN Cloud TTS declare their non-local behavior. The helper automatically cleans only the outbound synthesis-input copy and normalizes application-facing audio/text behavior while request/response plumbing differs below that boundary. |
|
|
83
84
|
| Inspect or manage raw Ollama models | `Arcane.ollama` or `/arcane/modules/Ollama.js` | **Native** desktop Core for management; narrower Android inference only | Wrapper method names, errors, streaming correlation, and admission are Arcane-controlled. Direct Ollama success envelopes are intentionally provider-native. |
|
|
84
85
|
| Use native terminal, installation, user, provisioning, or machine controls | matching `Arcane.*` namespace | **Native** and app/capability restricted | Calls and errors use the common bridge contract. Platform results can be host-specific and are marked in the method guide. |
|
|
85
86
|
|