arcane-os 0.46.0 → 0.47.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +1 -1
- package/browser-runtime/ai/browser-speech-recognition.mjs +495 -0
- package/browser-runtime/ai/browser-speech.mjs +1 -0
- package/docs/reference/ai/browser-speech.md +80 -2
- package/docs/reference/availability-and-normalization.md +2 -1
- package/docs/reference/inventory/package-api.json +14 -0
- package/docs/reference/runtime-components.md +23 -0
- package/docs/reference/runtime-modules.md +30 -2
- package/docs/reference/sdk-api.md +57 -0
- package/package.json +1 -1
- package/runtime/arcane/components/speech.html +141 -11
- package/runtime/arcane/components/voice-transcription.html +77 -8
- package/runtime/arcane/modules/AI.js +15 -4
- package/runtime/arcane/modules/AIProviderRuntime.js +118 -0
|
@@ -3134,6 +3134,20 @@
|
|
|
3134
3134
|
"protocol": "arcane-ai-provider/2 tts/synthesize",
|
|
3135
3135
|
"normalization": "Construction and activation make no adapter network requests. SDK initial POSTs bypass the provider-local js-queue; follow-up status/result/audio GETs use followUpQueue capacity and cadence through complete body consumption. Retry-After affects only its job. Whole-job maxConcurrentRequests remains independent. Local Kokoro and Whisper paths are unchanged. Applications own model, voice, credentials, activation, and queue settings."
|
|
3136
3136
|
},
|
|
3137
|
+
{
|
|
3138
|
+
"id": "browser-speech:createBrowserSpeechRecognitionProvider",
|
|
3139
|
+
"name": "createBrowserSpeechRecognitionProvider",
|
|
3140
|
+
"displayName": "createBrowserSpeechRecognitionProvider()",
|
|
3141
|
+
"kind": "function",
|
|
3142
|
+
"signature": "createBrowserSpeechRecognitionProvider({id,model,language}={})",
|
|
3143
|
+
"entrypoints": ["arcane-os/ai/browser-speech"],
|
|
3144
|
+
"primaryImport": "arcane-os/ai/browser-speech",
|
|
3145
|
+
"group": "Browser speech providers",
|
|
3146
|
+
"summary": "Creates an explicitly selected native browser speech recognition provider with gesture-started live capture.",
|
|
3147
|
+
"availability": "Browser or WebView exposing SpeechRecognition or webkitSpeechRecognition; construction remains available on unsupported hosts",
|
|
3148
|
+
"protocol": "arcane-ai-provider/2 stt with optional createCapture",
|
|
3149
|
+
"normalization": "Non-local service declaration, independent STT lifecycle, exact final segments, transient interim results, graceful stop/drain and native abort. No model download, storage, microphone on activation, file-transcription emulation, or automatic fallback."
|
|
3150
|
+
},
|
|
3137
3151
|
{
|
|
3138
3152
|
"id": "browser-speech:createBrowserKokoroProvider",
|
|
3139
3153
|
"name": "createBrowserKokoroProvider",
|
|
@@ -1629,6 +1629,16 @@ Events: `source-explanation-ready`, `source-explanation-save`, `source-explanati
|
|
|
1629
1629
|
|
|
1630
1630
|
## speech.html
|
|
1631
1631
|
|
|
1632
|
+
An explicitly selected native Web Speech provider uses gesture-started live
|
|
1633
|
+
capture instead of MediaRecorder. The shared control retains every final
|
|
1634
|
+
result and emits its existing completion event once with the exact combined
|
|
1635
|
+
text when native capture ends. The live display retains final words alongside
|
|
1636
|
+
interim hypotheses while speaking; interim hypotheses stay transient. Release/Stop
|
|
1637
|
+
finishes native recognition; cancellation, replacement and destruction abort
|
|
1638
|
+
it and suppress late submission. A persisted page hide stops live native capture
|
|
1639
|
+
without restarting it when the page returns. Existing Whisper and TTS paths
|
|
1640
|
+
remain unchanged.
|
|
1641
|
+
|
|
1632
1642
|
### Overview
|
|
1633
1643
|
|
|
1634
1644
|
Coordinates explicit speech-to-text activation, speech controls, transcription
|
|
@@ -2083,6 +2093,19 @@ Native methods remain subject to the bound app's capabilities. [Deep protocol de
|
|
|
2083
2093
|
|
|
2084
2094
|
### Continuous capture and controls
|
|
2085
2095
|
|
|
2096
|
+
An explicitly selected native Web Speech provider uses the same ordered
|
|
2097
|
+
transcript/save queue in either capture mode. Each complete final text segment
|
|
2098
|
+
skips file transcription and enters the existing append/save stages; interim
|
|
2099
|
+
text stays in the transient `interim` part and never reaches a save or success
|
|
2100
|
+
event. Final segments waiting behind a save stay visible there until appended
|
|
2101
|
+
to the main transcript. Native `stopRecording()` returns the drain promise, including final
|
|
2102
|
+
results emitted during native stop. Browser `end` stops capture without an
|
|
2103
|
+
automatic restart. Provider replacement stops native capture while already
|
|
2104
|
+
accepted final segments retain their ordered saves. Explicit Cancel, transcript
|
|
2105
|
+
replacement and destruction retain the existing supersession behavior.
|
|
2106
|
+
This path does not require AudioWorklet or MediaRecorder. A supplied custom
|
|
2107
|
+
`transcribe(file,context)` callback keeps the existing audio-clip route.
|
|
2108
|
+
|
|
2086
2109
|
`configure({capture:{mode:'continuous'}})` opts into the SDK's
|
|
2087
2110
|
[`ContinuousVoiceCapture`](runtime-modules.md#continuousvoicecapturejs) owner.
|
|
2088
2111
|
The default capture settings are `preRollMs:1500`, `quietMs:2000`,
|
|
@@ -275,8 +275,9 @@ tool declarations, emitted tool calls, or callback ordering.
|
|
|
275
275
|
`configureSpeechProviders({stt,tts})` commits only the two speech routes and
|
|
276
276
|
leaves the current LLM route and sticky lifecycle record unchanged. Both speech
|
|
277
277
|
roles must be unloaded and own no request, load, unload, or dispose operation.
|
|
278
|
-
STT remains local-only (`AI_STT_DEVICE_ONLY` for
|
|
279
|
-
|
|
278
|
+
File-based STT remains local-only (`AI_STT_DEVICE_ONLY` for an unsupported
|
|
279
|
+
remote file selection). An explicitly selected native live-capture STT provider
|
|
280
|
+
or TTS provider may declare `localOnly:false`.
|
|
280
281
|
`transitionSpeechProviders({stt,tts})` stops queued audio, explicitly unloads
|
|
281
282
|
only STT and TTS, then commits that same closed speech route record. Neither
|
|
282
283
|
method loads a model, selects a fallback, or changes caller-owned model or voice
|
|
@@ -302,6 +303,13 @@ cancels that pending unmute. The application keeps model, voice, and credential
|
|
|
302
303
|
selection. See [DigitalOcean FAL speech](ai/browser-speech.md#digitalocean-fal-text-to-speech)
|
|
303
304
|
for the public remote adapter and completed-job audio transport.
|
|
304
305
|
|
|
306
|
+
`supportsTranscriptionCapture()` reports whether the selected STT provider
|
|
307
|
+
offers native live capture. `createTranscriptionCapture(options={})` delegates
|
|
308
|
+
to the provider runtime's owned capture boundary below and returns `null` for
|
|
309
|
+
file-only providers. Shared speech controls select it automatically; apps keep
|
|
310
|
+
using their existing controls and completion/save callbacks. See
|
|
311
|
+
[native browser recognition](ai/browser-speech.md#native-browser-speech-recognition).
|
|
312
|
+
|
|
305
313
|
`startProviders({startLanguageModel=true,startMuted=true,startTranscription=false,signal=null}={})`
|
|
306
314
|
starts provider-owned text chat without requesting an STT load by default.
|
|
307
315
|
Callers selecting a browser-WASM LLM pass `startLanguageModel:false` so it
|
|
@@ -961,6 +969,8 @@ The singleton exposes read-only `protocol`, `configured`, and `speechMuted`;
|
|
|
961
969
|
`request(role,options={},preparation={})`;
|
|
962
970
|
`chat(payload,options={})`; `stream(payload,options={})`;
|
|
963
971
|
`transcribe(payload,options={})`;
|
|
972
|
+
`supportsTranscriptionCapture(providerId=null)`;
|
|
973
|
+
`createTranscriptionCapture(options={})`;
|
|
964
974
|
`synthesize(payload,options={},preparation={})`; and
|
|
965
975
|
`setSpeechMuted(muted)`. Provider payloads must be data-only; callbacks,
|
|
966
976
|
accessors, symbols, and cycles are rejected at the provider boundary.
|
|
@@ -1099,6 +1109,24 @@ cleanup are normalized, while the
|
|
|
1099
1109
|
selected provider retains its own capability, permission, download, and model
|
|
1100
1110
|
requirements. [Deep protocol details](protocols.md#portable-ai-provider-runtime).
|
|
1101
1111
|
|
|
1112
|
+
STT providers may additionally implement `createCapture(options)`. Registration
|
|
1113
|
+
retains that optional method without changing the required provider/2 methods.
|
|
1114
|
+
`supportsTranscriptionCapture(providerId=null)` checks the current selection by
|
|
1115
|
+
default or a specified registered STT provider without starting it.
|
|
1116
|
+
`createTranscriptionCapture({language,continuous,onSegment,onInterim,onState,onError})`
|
|
1117
|
+
returns `null` for a provider without that method; otherwise it returns
|
|
1118
|
+
`{start,stop,cancel,destroy,done}`. Creation has no microphone side effect.
|
|
1119
|
+
`start({signal})` requires the same selected ready provider with an idle STT
|
|
1120
|
+
lane and invokes its capture start synchronously in the caller's gesture.
|
|
1121
|
+
Capture occupies one active STT request until its `done` promise settles;
|
|
1122
|
+
ordinary file requests remain queued, and unload/dispose/cancel use the same
|
|
1123
|
+
owned request cancellation lifecycle. Stop drains native final recognition;
|
|
1124
|
+
cancel suppresses late callbacks. Provider callbacks carry exact final
|
|
1125
|
+
`{text,sequence}` segments, transient `{text}` interim results, capture state,
|
|
1126
|
+
or complete errors. The browser provider's `done` resolves a success boolean;
|
|
1127
|
+
runtime cleanup errors remain observable as rejection. No callback is inserted
|
|
1128
|
+
into a data-only transcription request payload.
|
|
1129
|
+
|
|
1102
1130
|
### Example
|
|
1103
1131
|
|
|
1104
1132
|
```javascript
|
|
@@ -169,6 +169,7 @@ browser map are cataloged separately in [Runtime modules](runtime-modules.md).
|
|
|
169
169
|
| `createBrowserSpeechAuthority()` | function | `arcane-os/ai/browser-speech` | Browser speech providers | Browser descriptor construction; use requires the selected storage and provider Web APIs |
|
|
170
170
|
| `createBrowserWasmLlmProvider()` | function | `arcane-os/ai/browser-wasm` | Browser-WASM local AI | Browser context with WebAssembly and OPFS/DBOPFS; WebGPU by default or explicit CPU with gpuLayers:0 |
|
|
171
171
|
| `createBrowserWhisperProvider()` | function | `arcane-os/ai/browser-speech` | Browser speech providers | Browser with Workers, object URLs, caller-selected Whisper runtime/model artifacts, and an SDK DBOPFS speech store |
|
|
172
|
+
| `createBrowserSpeechRecognitionProvider()` | function | `arcane-os/ai/browser-speech` | Browser speech providers | Browser or WebView with native SpeechRecognition; construction and catalog remain available on unsupported hosts |
|
|
172
173
|
| `createCanonicalUstarHeader()` | function | `arcane-os` | Packaging and release bundles | Node |
|
|
173
174
|
| `createDbopfsModelStore()` | function | `arcane-os/ai/browser-wasm` | Browser-WASM local AI | Browser with a ready DBOPFS instance and OPFS |
|
|
174
175
|
| `getBrowserDeviceClass()` | function | `arcane-os/browser-device` | Browser device settings | Node and Browser; synchronous identity hint with no model, storage, or GPU operation |
|
|
@@ -7185,6 +7186,62 @@ async function transcribeAfterUserChoice(audioBlob) {
|
|
|
7185
7186
|
}
|
|
7186
7187
|
```
|
|
7187
7188
|
|
|
7189
|
+
## createBrowserSpeechRecognitionProvider()
|
|
7190
|
+
|
|
7191
|
+
### Overview
|
|
7192
|
+
|
|
7193
|
+
Creates an explicitly selected provider/2 STT adapter for the browser's native
|
|
7194
|
+
`SpeechRecognition` or `webkitSpeechRecognition` service. It declares
|
|
7195
|
+
`localOnly:false` because recognition may use a remote browser service. It does
|
|
7196
|
+
not change Whisper, TTS, typing, stored preferences, or any default selection.
|
|
7197
|
+
|
|
7198
|
+
### Signature and result
|
|
7199
|
+
|
|
7200
|
+
```text
|
|
7201
|
+
createBrowserSpeechRecognitionProvider({id,model,language}={})
|
|
7202
|
+
```
|
|
7203
|
+
|
|
7204
|
+
The application supplies its provider `id` and `model:{id}`; optional `language`
|
|
7205
|
+
is passed to native recognition without changing transcript text. The provider
|
|
7206
|
+
implements the required `catalog`, `inspect`, `status`, `load`, `request`,
|
|
7207
|
+
`unload`, and `dispose` methods plus `createCapture(options)`. There is no SDK
|
|
7208
|
+
model download or storage. Construction/catalog do not require native support;
|
|
7209
|
+
inspect/load report actual browser availability without opening a microphone.
|
|
7210
|
+
File transcription requests are unsupported for this provider.
|
|
7211
|
+
|
|
7212
|
+
`createCapture({language,continuous,onSegment,onInterim,onState,onError})`
|
|
7213
|
+
returns a single-use `{start,stop,cancel,destroy,done}` handle. Start must be
|
|
7214
|
+
called directly from a user gesture and returns a promise for startup success.
|
|
7215
|
+
Final callbacks carry exact `{text,sequence}` values once per final result;
|
|
7216
|
+
interim `{text}` stays transient. Stop preserves pending native final results
|
|
7217
|
+
through `end`; cancellation aborts recognition and suppresses late text.
|
|
7218
|
+
`done` resolves `true` for normal native end and `false` for cancellation or
|
|
7219
|
+
failure. Unload/dispose cancel owned capture.
|
|
7220
|
+
The shared runtime's `createTranscriptionCapture()` additionally owns STT busy
|
|
7221
|
+
state and lifecycle replacement. See the
|
|
7222
|
+
[complete native capture contract](ai/browser-speech.md#native-browser-speech-recognition).
|
|
7223
|
+
|
|
7224
|
+
### Availability and normalization
|
|
7225
|
+
|
|
7226
|
+
Conditional browser/native-WebView support on Windows, Linux, macOS and
|
|
7227
|
+
Android, determined by the actual browser API and permission/service outcome,
|
|
7228
|
+
not by operating-system name. Missing API, denied microphone permission and
|
|
7229
|
+
service errors remain honest STT failures without disabling TTS or typing.
|
|
7230
|
+
No Core, Whisper, or cloud-provider fallback is inferred.
|
|
7231
|
+
|
|
7232
|
+
### Example
|
|
7233
|
+
|
|
7234
|
+
```javascript
|
|
7235
|
+
import {createBrowserSpeechRecognitionProvider} from 'arcane-os/ai/browser-speech';
|
|
7236
|
+
|
|
7237
|
+
async function selectNativeRecognition(ai, signal) {
|
|
7238
|
+
const provider = createBrowserSpeechRecognitionProvider(
|
|
7239
|
+
{id: 'WEB_SPEECH', model: {id: 'WEB_SPEECH'}}
|
|
7240
|
+
);
|
|
7241
|
+
return ai.configureSpeechProvider('stt', provider, {modelId: 'WEB_SPEECH', signal});
|
|
7242
|
+
}
|
|
7243
|
+
```
|
|
7244
|
+
|
|
7188
7245
|
## createTwinCloudTTSProvider()
|
|
7189
7246
|
|
|
7190
7247
|
### Overview
|
package/package.json
CHANGED
|
@@ -191,6 +191,7 @@
|
|
|
191
191
|
}
|
|
192
192
|
);
|
|
193
193
|
const aiModuleImport = import('../modules/AI.js');
|
|
194
|
+
const providerRuntimeImport = import('../modules/AIProviderRuntime.js');
|
|
194
195
|
const {
|
|
195
196
|
requestAIRuntimeIntent,
|
|
196
197
|
subscribeAIRuntimeState
|
|
@@ -200,6 +201,8 @@
|
|
|
200
201
|
formatAIRuntimeProgress
|
|
201
202
|
} = await import('../modules/ComponentContracts.js');
|
|
202
203
|
const {getBrowserDeviceClass} = await browserDeviceImport;
|
|
204
|
+
const {getAIProviderRuntime} = await providerRuntimeImport;
|
|
205
|
+
const providerRuntime = getAIProviderRuntime();
|
|
203
206
|
|
|
204
207
|
const host = this;
|
|
205
208
|
const deviceClass = getBrowserDeviceClass();
|
|
@@ -237,6 +240,7 @@
|
|
|
237
240
|
let microphonePermissionStatus = null;
|
|
238
241
|
let microphonePermissionListener = null;
|
|
239
242
|
let captureSession = null;
|
|
243
|
+
let nativeCaptureSession = null;
|
|
240
244
|
let recordingRequested = false;
|
|
241
245
|
let recordingGeneration = 0;
|
|
242
246
|
let transcriptionGeneration = 0;
|
|
@@ -362,8 +366,8 @@
|
|
|
362
366
|
host.setMuted = setMuted;
|
|
363
367
|
host.availability = {
|
|
364
368
|
microphone: transcriptionEnabled && Boolean(
|
|
365
|
-
navigator.mediaDevices?.getUserMedia
|
|
366
|
-
|
|
369
|
+
(navigator.mediaDevices?.getUserMedia && globalThis.MediaRecorder)
|
|
370
|
+
|| globalThis.SpeechRecognition || globalThis.webkitSpeechRecognition
|
|
367
371
|
),
|
|
368
372
|
stt: false,
|
|
369
373
|
tts: false
|
|
@@ -626,24 +630,25 @@
|
|
|
626
630
|
}
|
|
627
631
|
const transcriptionReady = sttRole.state === 'ready';
|
|
628
632
|
const microphoneReady = host.availability.microphone
|
|
629
|
-
&&
|
|
630
|
-
|
|
633
|
+
&& (providerRuntime.supportsTranscriptionCapture()
|
|
634
|
+
|| (is.function(globalThis.MediaRecorder)
|
|
635
|
+
&& is.function(navigator.mediaDevices?.getUserMedia)));
|
|
631
636
|
const activationVisible = sttActivationController.visible;
|
|
632
637
|
recordButton.hidden = activationVisible;
|
|
633
638
|
recordButton.disabled = !transcriptionReady
|
|
634
639
|
|| !microphoneReady
|
|
635
|
-
|| sttRole.busy
|
|
640
|
+
|| (sttRole.busy && !nativeCaptureSession)
|
|
636
641
|
|| transcriptionActive
|
|
637
642
|
|| destroyed;
|
|
638
643
|
stopButton.disabled = destroyed
|
|
639
644
|
|| !transcriptionReady
|
|
640
|
-
|| sttRole.busy
|
|
641
|
-
|| captureSession?.recorder.state !== 'recording';
|
|
645
|
+
|| (sttRole.busy && !nativeCaptureSession)
|
|
646
|
+
|| (!nativeCaptureSession && captureSession?.recorder.state !== 'recording');
|
|
642
647
|
recordButton.title = transcriptionButtonTitle();
|
|
643
648
|
recordButton.setAttribute('aria-label', recordButton.title);
|
|
644
649
|
recordButton.setAttribute(
|
|
645
650
|
'aria-pressed',
|
|
646
|
-
String(recordingRequested || Boolean(captureSession))
|
|
651
|
+
String(recordingRequested || Boolean(captureSession) || Boolean(nativeCaptureSession))
|
|
647
652
|
);
|
|
648
653
|
sttActivationButton.hidden = !activationVisible;
|
|
649
654
|
sttActivationButton.disabled = destroyed
|
|
@@ -1288,21 +1293,23 @@
|
|
|
1288
1293
|
}
|
|
1289
1294
|
}
|
|
1290
1295
|
);
|
|
1291
|
-
transcriptionOperationId = null;
|
|
1296
|
+
if (transcriptionOperationId === operationId) transcriptionOperationId = null;
|
|
1292
1297
|
return accepted;
|
|
1293
1298
|
}
|
|
1294
1299
|
|
|
1295
1300
|
async function record() {
|
|
1301
|
+
const liveCapture = providerRuntime.supportsTranscriptionCapture();
|
|
1296
1302
|
if (
|
|
1297
1303
|
!transcriptionEnabled
|
|
1298
1304
|
|| destroyed
|
|
1299
1305
|
|| sttRole.state !== 'ready'
|
|
1300
1306
|
|| sttRole.busy
|
|
1301
1307
|
|| !host.availability.microphone
|
|
1302
|
-
|| !is.function(globalThis.MediaRecorder)
|
|
1303
|
-
|
|
1308
|
+
|| (!liveCapture && (!is.function(globalThis.MediaRecorder)
|
|
1309
|
+
|| !is.function(navigator.mediaDevices?.getUserMedia)))
|
|
1304
1310
|
|| recordingRequested
|
|
1305
1311
|
|| captureSession
|
|
1312
|
+
|| nativeCaptureSession
|
|
1306
1313
|
|| transcriptionActive
|
|
1307
1314
|
) {
|
|
1308
1315
|
resetRecordPress();
|
|
@@ -1322,6 +1329,8 @@
|
|
|
1322
1329
|
renderControls();
|
|
1323
1330
|
renderStatus();
|
|
1324
1331
|
|
|
1332
|
+
if (liveCapture) return startNativeRecognition(generation, operationId);
|
|
1333
|
+
|
|
1325
1334
|
let stream;
|
|
1326
1335
|
try {
|
|
1327
1336
|
stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
@@ -1461,6 +1470,111 @@
|
|
|
1461
1470
|
}
|
|
1462
1471
|
}
|
|
1463
1472
|
|
|
1473
|
+
function startNativeRecognition(generation, operationId) {
|
|
1474
|
+
const session = {
|
|
1475
|
+
generation,
|
|
1476
|
+
operationId,
|
|
1477
|
+
controller: new AbortController(),
|
|
1478
|
+
capture: null,
|
|
1479
|
+
text: [],
|
|
1480
|
+
interim: '',
|
|
1481
|
+
cancelled: false,
|
|
1482
|
+
error: null
|
|
1483
|
+
};
|
|
1484
|
+
nativeCaptureSession = session;
|
|
1485
|
+
function currentNativeCapture() {
|
|
1486
|
+
return !destroyed && !session.cancelled
|
|
1487
|
+
&& nativeCaptureSession === session
|
|
1488
|
+
&& generation === recordingGeneration;
|
|
1489
|
+
}
|
|
1490
|
+
function displayNativeRecognitionText() {
|
|
1491
|
+
if (!currentNativeCapture()) return;
|
|
1492
|
+
localStatus = {
|
|
1493
|
+
message: session.text.join('') + session.interim || 'Listening. Release to finish.',
|
|
1494
|
+
tone: '',
|
|
1495
|
+
owner: 'recording',
|
|
1496
|
+
generation
|
|
1497
|
+
};
|
|
1498
|
+
renderStatus();
|
|
1499
|
+
}
|
|
1500
|
+
async function finishNativeRecognition(error = null) {
|
|
1501
|
+
if (!currentNativeCapture()) return;
|
|
1502
|
+
nativeCaptureSession = null;
|
|
1503
|
+
recordingRequested = false;
|
|
1504
|
+
resetRecordPress();
|
|
1505
|
+
clearRecordingStatus(generation);
|
|
1506
|
+
if (error && error !== session.error) {
|
|
1507
|
+
session.error = error;
|
|
1508
|
+
reportTranscriptionError(error);
|
|
1509
|
+
}
|
|
1510
|
+
// Keep confirmed final words after a service/cleanup failure. Only
|
|
1511
|
+
// explicit cancellation or supersession abandons this submission.
|
|
1512
|
+
if (session.text.length && !destroyed && !session.cancelled
|
|
1513
|
+
&& generation === recordingGeneration) {
|
|
1514
|
+
transcriptionOperationId = operationId;
|
|
1515
|
+
await transcribe(session.text.join(''));
|
|
1516
|
+
} else if (transcriptionOperationId === operationId) {
|
|
1517
|
+
transcriptionOperationId = null;
|
|
1518
|
+
}
|
|
1519
|
+
renderControls();
|
|
1520
|
+
renderStatus();
|
|
1521
|
+
}
|
|
1522
|
+
try {
|
|
1523
|
+
session.capture = providerRuntime.createTranscriptionCapture(
|
|
1524
|
+
{
|
|
1525
|
+
continuous: true,
|
|
1526
|
+
onSegment: function retainFinalRecognition(segment) {
|
|
1527
|
+
if (!currentNativeCapture()) return;
|
|
1528
|
+
session.text.push(segment.text);
|
|
1529
|
+
displayNativeRecognitionText();
|
|
1530
|
+
},
|
|
1531
|
+
onInterim: function displayInterimRecognition(result) {
|
|
1532
|
+
if (!currentNativeCapture()) return;
|
|
1533
|
+
session.interim = result.text;
|
|
1534
|
+
displayNativeRecognitionText();
|
|
1535
|
+
},
|
|
1536
|
+
onError: function reportNativeRecognitionFailure(error) {
|
|
1537
|
+
if (!currentNativeCapture()) return;
|
|
1538
|
+
session.error = error;
|
|
1539
|
+
reportTranscriptionError(error);
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
);
|
|
1543
|
+
if (!session.capture) throw new Error('The selected live transcription service is unavailable.');
|
|
1544
|
+
session.capture.done.then(
|
|
1545
|
+
function nativeRecognitionCompleted() {
|
|
1546
|
+
return finishNativeRecognition();
|
|
1547
|
+
},
|
|
1548
|
+
finishNativeRecognition
|
|
1549
|
+
).catch(
|
|
1550
|
+
function reportNativeRecognitionSettlementFailure(error) {
|
|
1551
|
+
if (destroyed || generation !== recordingGeneration) return;
|
|
1552
|
+
reportTranscriptionError(error);
|
|
1553
|
+
}
|
|
1554
|
+
);
|
|
1555
|
+
// No awaited preparation precedes the browser's gesture-bound start.
|
|
1556
|
+
const starting = session.capture.start({signal: session.controller.signal});
|
|
1557
|
+
return Promise.resolve(starting).catch(
|
|
1558
|
+
function reportNativeRecognitionStartupFailure(error) {
|
|
1559
|
+
if (!currentNativeCapture()) return false;
|
|
1560
|
+
session.capture.cancel();
|
|
1561
|
+
nativeCaptureSession = null;
|
|
1562
|
+
recordingRequested = false;
|
|
1563
|
+
resetRecordPress();
|
|
1564
|
+
if (error !== session.error) reportTranscriptionError(error);
|
|
1565
|
+
return false;
|
|
1566
|
+
}
|
|
1567
|
+
);
|
|
1568
|
+
} catch (error) {
|
|
1569
|
+
session.capture?.cancel();
|
|
1570
|
+
nativeCaptureSession = null;
|
|
1571
|
+
recordingRequested = false;
|
|
1572
|
+
resetRecordPress();
|
|
1573
|
+
if (error !== session.error) reportTranscriptionError(error);
|
|
1574
|
+
return false;
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
|
|
1464
1578
|
function stopAfterSilence() {
|
|
1465
1579
|
clearSilenceTimer();
|
|
1466
1580
|
stop();
|
|
@@ -1483,6 +1597,12 @@
|
|
|
1483
1597
|
}
|
|
1484
1598
|
|
|
1485
1599
|
function stop() {
|
|
1600
|
+
if (nativeCaptureSession) {
|
|
1601
|
+
recordingRequested = false;
|
|
1602
|
+
nativeCaptureSession.capture.stop();
|
|
1603
|
+
renderControls();
|
|
1604
|
+
return true;
|
|
1605
|
+
}
|
|
1486
1606
|
const pendingCaptureStart = recordingRequested && !captureSession;
|
|
1487
1607
|
const generation = recordingGeneration;
|
|
1488
1608
|
const operationId = transcriptionOperationId;
|
|
@@ -1703,6 +1823,7 @@
|
|
|
1703
1823
|
function cancelSTTOperation(reason) {
|
|
1704
1824
|
const hadActiveOperation = recordingRequested
|
|
1705
1825
|
|| Boolean(captureSession)
|
|
1826
|
+
|| Boolean(nativeCaptureSession)
|
|
1706
1827
|
|| transcriptionActive;
|
|
1707
1828
|
recordingGeneration += 1;
|
|
1708
1829
|
transcriptionGeneration += 1;
|
|
@@ -1713,6 +1834,14 @@
|
|
|
1713
1834
|
clearSilenceTimer();
|
|
1714
1835
|
resetRecordPress();
|
|
1715
1836
|
|
|
1837
|
+
if (nativeCaptureSession) {
|
|
1838
|
+
const session = nativeCaptureSession;
|
|
1839
|
+
nativeCaptureSession = null;
|
|
1840
|
+
session.cancelled = true;
|
|
1841
|
+
session.controller.abort();
|
|
1842
|
+
session.capture?.cancel();
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1716
1845
|
if (captureSession) {
|
|
1717
1846
|
captureSession.cancelled = true;
|
|
1718
1847
|
if (captureSession.recorder.state !== 'inactive') {
|
|
@@ -1769,6 +1898,7 @@
|
|
|
1769
1898
|
|
|
1770
1899
|
function handlePageHide(event) {
|
|
1771
1900
|
if (event.persisted === true) {
|
|
1901
|
+
if (nativeCaptureSession) stop();
|
|
1772
1902
|
return;
|
|
1773
1903
|
}
|
|
1774
1904
|
destroy();
|
|
@@ -82,6 +82,7 @@
|
|
|
82
82
|
<p id="description" class="description" part="description"></p>
|
|
83
83
|
<section id="transcript" class="transcript" part="transcript"></section>
|
|
84
84
|
<div class="transcription-feedback">
|
|
85
|
+
<p id="interim" class="status" part="interim" aria-live="polite" hidden></p>
|
|
85
86
|
<p id="status" class="status" part="status" role="status" aria-live="polite"></p>
|
|
86
87
|
<progress
|
|
87
88
|
id="sttActivationProgress"
|
|
@@ -136,16 +137,20 @@
|
|
|
136
137
|
{default:MD},
|
|
137
138
|
{appendTranscription,createSTTActivationController,normalizeVoiceOptions},
|
|
138
139
|
{requestAIRuntimeIntent,subscribeAIRuntimeState},
|
|
139
|
-
{default:ContinuousVoiceCapture}
|
|
140
|
+
{default:ContinuousVoiceCapture},
|
|
141
|
+
{getAIProviderRuntime}
|
|
140
142
|
]=await Promise.all([
|
|
141
143
|
import('../modules/MD.js'),
|
|
142
144
|
import('../modules/ComponentContracts.js'),
|
|
143
145
|
import('../modules/AIRuntimeState.js'),
|
|
144
|
-
import('../modules/ContinuousVoiceCapture.js')
|
|
146
|
+
import('../modules/ContinuousVoiceCapture.js'),
|
|
147
|
+
import('../modules/AIProviderRuntime.js')
|
|
145
148
|
]);
|
|
149
|
+
const providerRuntime=getAIProviderRuntime();
|
|
146
150
|
const description=host.shadowRoot.querySelector('#description');
|
|
147
151
|
const transcriptElement=host.shadowRoot.querySelector('#transcript');
|
|
148
152
|
const status=host.shadowRoot.querySelector('#status');
|
|
153
|
+
const interimElement=host.shadowRoot.querySelector('#interim');
|
|
149
154
|
const startButton=host.shadowRoot.querySelector('#start');
|
|
150
155
|
const sttActivationButton=host.shadowRoot.querySelector('#sttActivation');
|
|
151
156
|
const sttActivationProgress = host.shadowRoot.querySelector('#sttActivationProgress');
|
|
@@ -385,6 +390,7 @@
|
|
|
385
390
|
flushing:false,
|
|
386
391
|
processing:'idle',
|
|
387
392
|
queue:[],
|
|
393
|
+
interim:'',
|
|
388
394
|
failure:null,
|
|
389
395
|
captureError:null,
|
|
390
396
|
worker:null,
|
|
@@ -396,15 +402,25 @@
|
|
|
396
402
|
if(!isCurrentContinuousSession(session)||!session.listening){
|
|
397
403
|
return false;
|
|
398
404
|
}
|
|
399
|
-
|
|
405
|
+
const captureOptions={
|
|
400
406
|
...options.capture,
|
|
401
407
|
onSegment:function capturedSegment(clip){
|
|
402
408
|
if(!isCurrentContinuousSession(session)){
|
|
403
409
|
return;
|
|
404
410
|
}
|
|
405
|
-
session.queue.push({
|
|
411
|
+
session.queue.push({
|
|
412
|
+
...clip,
|
|
413
|
+
stage:is.string(clip.text)?'append':'transcribe',
|
|
414
|
+
segment:is.string(clip.text)?clip.text:null
|
|
415
|
+
});
|
|
416
|
+
renderInterimRecognition(session);
|
|
406
417
|
processContinuousQueue(session);
|
|
407
418
|
},
|
|
419
|
+
onInterim:function displayInterimRecognition(result){
|
|
420
|
+
if(!isCurrentContinuousSession(session)) return;
|
|
421
|
+
session.interim=result.text;
|
|
422
|
+
renderInterimRecognition(session);
|
|
423
|
+
},
|
|
408
424
|
onState:function captureStateChanged(nextState){
|
|
409
425
|
if(!isCurrentContinuousSession(session)){
|
|
410
426
|
return;
|
|
@@ -412,8 +428,11 @@
|
|
|
412
428
|
session.captureState=nextState;
|
|
413
429
|
if(nextState==='stopped'||nextState==='interrupted'){
|
|
414
430
|
session.listening=false;
|
|
431
|
+
session.interim='';
|
|
432
|
+
renderInterimRecognition(session);
|
|
415
433
|
}
|
|
416
434
|
renderContinuousState(session);
|
|
435
|
+
settleContinuousDrain(session);
|
|
417
436
|
},
|
|
418
437
|
onError:function captureFailed(error){
|
|
419
438
|
if(!isCurrentContinuousSession(session)){
|
|
@@ -425,7 +444,29 @@
|
|
|
425
444
|
reportContinuousError(session,'capture',error);
|
|
426
445
|
settleContinuousDrain(session);
|
|
427
446
|
}
|
|
428
|
-
}
|
|
447
|
+
};
|
|
448
|
+
session.capture=!options.transcribe
|
|
449
|
+
?providerRuntime.createTranscriptionCapture(captureOptions):null;
|
|
450
|
+
session.native=Boolean(session.capture);
|
|
451
|
+
if(!session.capture){
|
|
452
|
+
session.capture=new ContinuousVoiceCapture(captureOptions);
|
|
453
|
+
}else{
|
|
454
|
+
session.capture.done.then(function nativeRecognitionEnded(){
|
|
455
|
+
if(!isCurrentContinuousSession(session)) return;
|
|
456
|
+
session.listening=false;
|
|
457
|
+
session.interim='';
|
|
458
|
+
renderInterimRecognition(session);
|
|
459
|
+
if(!session.captureError) session.captureState='stopped';
|
|
460
|
+
renderContinuousState(session);
|
|
461
|
+
settleContinuousDrain(session);
|
|
462
|
+
}).catch(function nativeRecognitionCleanupFailed(error){
|
|
463
|
+
if(!isCurrentContinuousSession(session)) return;
|
|
464
|
+
session.captureError=error;
|
|
465
|
+
session.listening=false;
|
|
466
|
+
reportContinuousError(session,'capture',error);
|
|
467
|
+
settleContinuousDrain(session);
|
|
468
|
+
});
|
|
469
|
+
}
|
|
429
470
|
const started=await session.capture.start({
|
|
430
471
|
mediaConstraints:options.mediaConstraints,
|
|
431
472
|
signal:session.controller.signal
|
|
@@ -467,6 +508,17 @@
|
|
|
467
508
|
};
|
|
468
509
|
}
|
|
469
510
|
|
|
511
|
+
function renderInterimRecognition(session){
|
|
512
|
+
if(!isCurrentContinuousSession(session)) return;
|
|
513
|
+
const pending=[];
|
|
514
|
+
for(const clip of session.queue){
|
|
515
|
+
if(clip.stage==='append') pending.push(clip.segment);
|
|
516
|
+
}
|
|
517
|
+
const text=pending.join(options.separator)+session.interim;
|
|
518
|
+
interimElement.textContent=text;
|
|
519
|
+
interimElement.hidden=text.length===0;
|
|
520
|
+
}
|
|
521
|
+
|
|
470
522
|
function renderContinuousState(session){
|
|
471
523
|
if(!isCurrentContinuousSession(session)){
|
|
472
524
|
return;
|
|
@@ -598,7 +650,10 @@
|
|
|
598
650
|
throw new TypeError('Transcription must return the complete text as a string.');
|
|
599
651
|
}
|
|
600
652
|
clip.segment=result;
|
|
601
|
-
|
|
653
|
+
clip.stage='append';
|
|
654
|
+
}
|
|
655
|
+
if(clip.stage==='append'){
|
|
656
|
+
if(clip.segment.length===0){
|
|
602
657
|
session.queue.shift();
|
|
603
658
|
const detail={sequence:clip.sequence,transcript};
|
|
604
659
|
dispatchVoiceEvent('voice-transcription-empty',detail,{
|
|
@@ -606,10 +661,11 @@
|
|
|
606
661
|
});
|
|
607
662
|
continue;
|
|
608
663
|
}
|
|
609
|
-
clip.transcript=appendTranscription(transcript,
|
|
664
|
+
clip.transcript=appendTranscription(transcript,clip.segment,options.separator);
|
|
610
665
|
clip.stage='save';
|
|
611
666
|
transcript=clip.transcript;
|
|
612
667
|
renderTranscript();
|
|
668
|
+
renderInterimRecognition(session);
|
|
613
669
|
}
|
|
614
670
|
if(options.persist){
|
|
615
671
|
const save=options.onSave||(is.function(host.save)?host.save.bind(host):null);
|
|
@@ -740,6 +796,8 @@
|
|
|
740
796
|
return false;
|
|
741
797
|
}
|
|
742
798
|
continuousSession=null;
|
|
799
|
+
interimElement.textContent='';
|
|
800
|
+
interimElement.hidden=true;
|
|
743
801
|
session.controller.abort();
|
|
744
802
|
session.requestController?.abort();
|
|
745
803
|
session.capture?.cancel();
|
|
@@ -766,7 +824,8 @@
|
|
|
766
824
|
}
|
|
767
825
|
|
|
768
826
|
async function startRecording(){
|
|
769
|
-
if(options.capture.mode==='continuous'
|
|
827
|
+
if(options.capture.mode==='continuous'
|
|
828
|
+
||(!options.transcribe&&providerRuntime.supportsTranscriptionCapture())){
|
|
770
829
|
return startContinuousRecording();
|
|
771
830
|
}
|
|
772
831
|
if(!canStartVoiceRecording(sttRole,state,destroyed)){
|
|
@@ -1366,6 +1425,16 @@
|
|
|
1366
1425
|
sttActivationController.synchronize(sttRole);
|
|
1367
1426
|
if(continuousSession){
|
|
1368
1427
|
const session=continuousSession;
|
|
1428
|
+
if(session.native&&session.listening
|
|
1429
|
+
&&(sttRole.state!=='ready'
|
|
1430
|
+
||previousRole.providerId!==sttRole.providerId
|
|
1431
|
+
||previousRole.modelId!==sttRole.modelId)){
|
|
1432
|
+
session.capture?.cancel();
|
|
1433
|
+
session.listening=false;
|
|
1434
|
+
session.captureState='interrupted';
|
|
1435
|
+
session.interim='';
|
|
1436
|
+
renderInterimRecognition(session);
|
|
1437
|
+
}
|
|
1369
1438
|
if(['completing','complete'].includes(state)){
|
|
1370
1439
|
renderState();
|
|
1371
1440
|
return;
|